chore: vendor sglang v0.5.10 snapshot

third_party/sglang/sgl-model-gateway/src/routers/conversations/handlers.rs

@@ -0,0 +1,633 @@
+//! Conversation CRUD handlers for the /v1/conversations API - shared across routers
+
+use std::sync::Arc;
+
+use axum::{
+    http::StatusCode,
+    response::{IntoResponse, Response},
+    Json,
+};
+use chrono::Utc;
+use data_connector::{
+    Conversation, ConversationId, ConversationItem, ConversationItemId, ConversationItemStorage,
+    ConversationStorage, ListParams, NewConversation, NewConversationItem, SortOrder,
+};
+use serde_json::{json, Value};
+use tracing::{info, warn};
+
+use crate::routers::persistence_utils::item_to_json;
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+pub const MAX_METADATA_PROPERTIES: usize = 16;
+const MAX_ITEMS_PER_REQUEST: usize = 20;
+
+const SUPPORTED_ITEM_TYPES: &[&str] = &[
+    "message",
+    "reasoning",
+    "mcp_list_tools",
+    "mcp_call",
+    "item_reference",
+    "function_call",
+    "function_call_output",
+    "file_search_call",
+    "computer_call",
+    "computer_call_output",
+    "web_search_call",
+    "image_generation_call",
+    "code_interpreter_call",
+    "local_shell_call",
+    "local_shell_call_output",
+    "mcp_approval_request",
+    "mcp_approval_response",
+    "custom_tool_call",
+    "custom_tool_call_output",
+];
+
+const IMPLEMENTED_ITEM_TYPES: &[&str] = &[
+    "message",
+    "reasoning",
+    "mcp_list_tools",
+    "mcp_call",
+    "item_reference",
+];
+
+// ============================================================================
+// Error Response Helpers
+// ============================================================================
+
+fn bad_request(message: impl Into<String>) -> Response {
+    (
+        StatusCode::BAD_REQUEST,
+        Json(json!({"error": message.into()})),
+    )
+        .into_response()
+}
+
+fn not_found(message: impl Into<String>) -> Response {
+    (
+        StatusCode::NOT_FOUND,
+        Json(json!({"error": message.into()})),
+    )
+        .into_response()
+}
+
+fn internal_error(message: impl Into<String>) -> Response {
+    (
+        StatusCode::INTERNAL_SERVER_ERROR,
+        Json(json!({"error": message.into()})),
+    )
+        .into_response()
+}
+
+fn bad_request_structured(error_obj: Value) -> Response {
+    (StatusCode::BAD_REQUEST, Json(json!({"error": error_obj}))).into_response()
+}
+
+// ============================================================================
+// Storage Helpers
+// ============================================================================
+
+async fn ensure_conversation_exists(
+    storage: &Arc<dyn ConversationStorage>,
+    conv_id: &ConversationId,
+) -> Result<Conversation, Response> {
+    match storage.get_conversation(conv_id).await {
+        Ok(Some(conv)) => Ok(conv),
+        Ok(None) => Err(not_found("Conversation not found")),
+        Err(e) => Err(internal_error(format!("Failed to get conversation: {e}"))),
+    }
+}
+
+// ============================================================================
+// Metadata Operations
+// ============================================================================
+
+fn validate_metadata(value: &Value) -> Result<Option<serde_json::Map<String, Value>>, String> {
+    match value.get("metadata") {
+        Some(Value::Object(map)) => {
+            if map.len() > MAX_METADATA_PROPERTIES {
+                Err(format!(
+                    "metadata cannot have more than {MAX_METADATA_PROPERTIES} properties"
+                ))
+            } else {
+                Ok(Some(map.clone()))
+            }
+        }
+        Some(_) => Err("metadata must be an object".to_string()),
+        None => Ok(None),
+    }
+}
+
+fn apply_metadata_patches(
+    current: Option<serde_json::Map<String, Value>>,
+    body: &Value,
+) -> Result<Option<serde_json::Map<String, Value>>, String> {
+    let patch_map = match body.get("metadata") {
+        Some(Value::Object(map)) => map,
+        Some(_) => return Err("metadata must be an object".to_string()),
+        None => return Ok(current),
+    };
+
+    let mut result = current.unwrap_or_default();
+    for (k, v) in patch_map {
+        if v.is_null() {
+            result.remove(k);
+        } else {
+            result.insert(k.clone(), v.clone());
+        }
+    }
+
+    if result.len() > MAX_METADATA_PROPERTIES {
+        return Err(format!(
+            "metadata cannot have more than {MAX_METADATA_PROPERTIES} properties"
+        ));
+    }
+
+    Ok(if result.is_empty() {
+        None
+    } else {
+        Some(result)
+    })
+}
+
+// ============================================================================
+// Conversation CRUD Handlers
+// ============================================================================
+
+pub async fn create_conversation(storage: &Arc<dyn ConversationStorage>, body: Value) -> Response {
+    let metadata = match validate_metadata(&body) {
+        Ok(m) => m,
+        Err(msg) => return bad_request(msg),
+    };
+
+    let new_conv = NewConversation { id: None, metadata };
+
+    match storage.create_conversation(new_conv).await {
+        Ok(conversation) => {
+            info!(conversation_id = %conversation.id.0, "Created conversation");
+            (StatusCode::OK, Json(conversation_to_json(&conversation))).into_response()
+        }
+        Err(e) => internal_error(format!("Failed to create conversation: {e}")),
+    }
+}
+
+pub async fn get_conversation(storage: &Arc<dyn ConversationStorage>, conv_id: &str) -> Response {
+    let conversation_id = ConversationId::from(conv_id);
+
+    match storage.get_conversation(&conversation_id).await {
+        Ok(Some(conversation)) => {
+            (StatusCode::OK, Json(conversation_to_json(&conversation))).into_response()
+        }
+        Ok(None) => not_found("Conversation not found"),
+        Err(e) => internal_error(format!("Failed to get conversation: {e}")),
+    }
+}
+
+pub async fn update_conversation(
+    storage: &Arc<dyn ConversationStorage>,
+    conv_id: &str,
+    body: Value,
+) -> Response {
+    let conversation_id = ConversationId::from(conv_id);
+
+    let current = match ensure_conversation_exists(storage, &conversation_id).await {
+        Ok(c) => c,
+        Err(response) => return response,
+    };
+
+    let final_metadata = match apply_metadata_patches(current.metadata.clone(), &body) {
+        Ok(m) => m,
+        Err(msg) => return bad_request(msg),
+    };
+
+    match storage
+        .update_conversation(&conversation_id, final_metadata)
+        .await
+    {
+        Ok(Some(conversation)) => {
+            info!(conversation_id = %conversation_id.0, "Updated conversation");
+            (StatusCode::OK, Json(conversation_to_json(&conversation))).into_response()
+        }
+        Ok(None) => not_found("Conversation not found"),
+        Err(e) => internal_error(format!("Failed to update conversation: {e}")),
+    }
+}
+
+pub async fn delete_conversation(
+    storage: &Arc<dyn ConversationStorage>,
+    conv_id: &str,
+) -> Response {
+    let conversation_id = ConversationId::from(conv_id);
+
+    if let Err(response) = ensure_conversation_exists(storage, &conversation_id).await {
+        return response;
+    }
+
+    match storage.delete_conversation(&conversation_id).await {
+        Ok(_) => {
+            info!(conversation_id = %conversation_id.0, "Deleted conversation");
+            (
+                StatusCode::OK,
+                Json(json!({
+                    "id": conversation_id.0,
+                    "object": "conversation.deleted",
+                    "deleted": true
+                })),
+            )
+                .into_response()
+        }
+        Err(e) => internal_error(format!("Failed to delete conversation: {e}")),
+    }
+}
+
+// ============================================================================
+// Conversation Item Handlers
+// ============================================================================
+
+pub async fn list_conversation_items(
+    conversation_storage: &Arc<dyn ConversationStorage>,
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conv_id: &str,
+    limit: Option<usize>,
+    order: Option<&str>,
+    after: Option<&str>,
+) -> Response {
+    let conversation_id = ConversationId::from(conv_id);
+
+    if let Err(response) = ensure_conversation_exists(conversation_storage, &conversation_id).await
+    {
+        return response;
+    }
+
+    let limit = limit.unwrap_or(100);
+    let order = match order {
+        Some("asc") => SortOrder::Asc,
+        _ => SortOrder::Desc,
+    };
+
+    let params = ListParams {
+        limit,
+        order,
+        after: after.map(String::from),
+    };
+
+    match item_storage.list_items(&conversation_id, params).await {
+        Ok(items) => {
+            let item_values: Vec<Value> = items
+                .iter()
+                .map(|item| {
+                    let mut item_json = item_to_json(item);
+                    if let Some(obj) = item_json.as_object_mut() {
+                        obj.insert("created_at".to_string(), json!(item.created_at));
+                    }
+                    item_json
+                })
+                .collect();
+
+            (
+                StatusCode::OK,
+                Json(json!({
+                    "object": "list",
+                    "data": item_values,
+                    "has_more": items.len() == limit,
+                    "first_id": items.first().map(|item| &item.id.0),
+                    "last_id": items.last().map(|item| &item.id.0),
+                })),
+            )
+                .into_response()
+        }
+        Err(e) => internal_error(format!("Failed to list items: {e}")),
+    }
+}
+
+pub async fn create_conversation_items(
+    conversation_storage: &Arc<dyn ConversationStorage>,
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conv_id: &str,
+    body: Value,
+) -> Response {
+    let conversation_id = ConversationId::from(conv_id);
+
+    if let Err(response) = ensure_conversation_exists(conversation_storage, &conversation_id).await
+    {
+        return response;
+    }
+
+    let items_array = match body.get("items").and_then(|v| v.as_array()) {
+        Some(arr) => arr,
+        None => return bad_request("Missing or invalid 'items' field"),
+    };
+
+    if items_array.len() > MAX_ITEMS_PER_REQUEST {
+        return bad_request(format!(
+            "Cannot add more than {MAX_ITEMS_PER_REQUEST} items at a time"
+        ));
+    }
+
+    let mut created_items = Vec::new();
+    let mut warnings = Vec::new();
+    let added_at = Utc::now();
+
+    for item_val in items_array {
+        match process_item(item_storage, &conversation_id, item_val, added_at).await {
+            Ok((item_json, warning)) => {
+                created_items.push(item_json);
+                if let Some(w) = warning {
+                    warnings.push(w);
+                }
+            }
+            Err(response) => return response,
+        }
+    }
+
+    let mut response = json!({
+        "object": "list",
+        "data": created_items,
+        "first_id": created_items.first().and_then(|v| v.get("id")),
+        "last_id": created_items.last().and_then(|v| v.get("id")),
+        "has_more": false
+    });
+
+    if !warnings.is_empty() {
+        if let Some(obj) = response.as_object_mut() {
+            obj.insert("warnings".to_string(), json!(warnings));
+        }
+    }
+
+    (StatusCode::OK, Json(response)).into_response()
+}
+
+/// Process a single item for creation/linking
+async fn process_item(
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conversation_id: &ConversationId,
+    item_val: &Value,
+    added_at: chrono::DateTime<Utc>,
+) -> Result<(Value, Option<String>), Response> {
+    let item_type = item_val
+        .get("type")
+        .and_then(|v| v.as_str())
+        .unwrap_or("message");
+
+    // Handle item_reference specially - just link existing item
+    if item_type == "item_reference" {
+        return process_item_reference(item_storage, conversation_id, item_val, added_at).await;
+    }
+
+    let user_provided_id = item_val.get("id").and_then(|v| v.as_str());
+
+    let (item, warning) = if let Some(id_str) = user_provided_id {
+        process_item_with_id(item_storage, conversation_id, item_val, id_str).await?
+    } else {
+        process_new_item(item_storage, item_val).await?
+    };
+
+    // Link item to conversation
+    if let Err(e) = item_storage
+        .link_item(conversation_id, &item.id, added_at)
+        .await
+    {
+        warn!("Failed to link item {}: {}", item.id.0, e);
+    }
+
+    Ok((item_to_json(&item), warning))
+}
+
+/// Process an item_reference - link an existing item to the conversation
+async fn process_item_reference(
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conversation_id: &ConversationId,
+    item_val: &Value,
+    added_at: chrono::DateTime<Utc>,
+) -> Result<(Value, Option<String>), Response> {
+    let ref_id = item_val
+        .get("id")
+        .and_then(|v| v.as_str())
+        .ok_or_else(|| bad_request("item_reference requires 'id' field"))?;
+
+    let item_id = ConversationItemId::from(ref_id);
+
+    let existing_item = match item_storage.get_item(&item_id).await {
+        Ok(Some(item)) => item,
+        Ok(None) => return Err(not_found(format!("Referenced item '{ref_id}' not found"))),
+        Err(e) => {
+            return Err(internal_error(format!(
+                "Failed to get referenced item: {e}"
+            )))
+        }
+    };
+
+    if let Err(e) = item_storage
+        .link_item(conversation_id, &existing_item.id, added_at)
+        .await
+    {
+        warn!("Failed to link item {}: {}", existing_item.id.0, e);
+    }
+
+    Ok((item_to_json(&existing_item), None))
+}
+
+/// Process an item with a user-provided ID
+async fn process_item_with_id(
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conversation_id: &ConversationId,
+    item_val: &Value,
+    id_str: &str,
+) -> Result<(ConversationItem, Option<String>), Response> {
+    let item_id = ConversationItemId::from(id_str);
+
+    // Check if already linked
+    let is_linked = item_storage
+        .is_item_linked(conversation_id, &item_id)
+        .await
+        .map_err(|e| internal_error(format!("Failed to check item link: {e}")))?;
+
+    if is_linked {
+        return Err(bad_request_structured(json!({
+            "message": "Item already in conversation",
+            "type": "invalid_request_error",
+            "param": "items",
+            "code": "item_already_in_conversation"
+        })));
+    }
+
+    // Check if item exists globally
+    match item_storage.get_item(&item_id).await {
+        Ok(Some(existing)) => Ok((existing, None)),
+        Ok(None) => {
+            // Create new item with the provided ID
+            let (mut new_item, warning) = parse_item_from_value(item_val).map_err(bad_request)?;
+            new_item.id = Some(item_id);
+
+            let created = item_storage
+                .create_item(new_item)
+                .await
+                .map_err(|e| internal_error(format!("Failed to create item: {e}")))?;
+
+            Ok((created, warning))
+        }
+        Err(e) => Err(internal_error(format!(
+            "Failed to check item existence: {e}"
+        ))),
+    }
+}
+
+/// Process a new item without a user-provided ID
+async fn process_new_item(
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    item_val: &Value,
+) -> Result<(ConversationItem, Option<String>), Response> {
+    let (new_item, warning) = parse_item_from_value(item_val).map_err(bad_request)?;
+
+    let created = item_storage
+        .create_item(new_item)
+        .await
+        .map_err(|e| internal_error(format!("Failed to create item: {e}")))?;
+
+    Ok((created, warning))
+}
+
+pub async fn get_conversation_item(
+    conversation_storage: &Arc<dyn ConversationStorage>,
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conv_id: &str,
+    item_id: &str,
+    _include: Option<Vec<String>>,
+) -> Response {
+    let conversation_id = ConversationId::from(conv_id);
+    let item_id = ConversationItemId::from(item_id);
+
+    if let Err(response) = ensure_conversation_exists(conversation_storage, &conversation_id).await
+    {
+        return response;
+    }
+
+    let is_linked = match item_storage
+        .is_item_linked(&conversation_id, &item_id)
+        .await
+    {
+        Ok(linked) => linked,
+        Err(e) => return internal_error(format!("Failed to check item link: {e}")),
+    };
+
+    if !is_linked {
+        return not_found("Item not found in this conversation");
+    }
+
+    match item_storage.get_item(&item_id).await {
+        Ok(Some(item)) => (StatusCode::OK, Json(item_to_json(&item))).into_response(),
+        Ok(None) => not_found("Item not found"),
+        Err(e) => internal_error(format!("Failed to get item: {e}")),
+    }
+}
+
+pub async fn delete_conversation_item(
+    conversation_storage: &Arc<dyn ConversationStorage>,
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conv_id: &str,
+    item_id: &str,
+) -> Response {
+    let conversation_id = ConversationId::from(conv_id);
+    let item_id = ConversationItemId::from(item_id);
+
+    let conversation =
+        match ensure_conversation_exists(conversation_storage, &conversation_id).await {
+            Ok(conv) => conv,
+            Err(response) => return response,
+        };
+
+    match item_storage.delete_item(&conversation_id, &item_id).await {
+        Ok(_) => {
+            info!(
+                conversation_id = %conversation_id.0,
+                item_id = %item_id.0,
+                "Deleted conversation item"
+            );
+            (StatusCode::OK, Json(conversation_to_json(&conversation))).into_response()
+        }
+        Err(e) => internal_error(format!("Failed to delete item: {e}")),
+    }
+}
+
+// ============================================================================
+// Parsing and Serialization
+// ============================================================================
+
+fn parse_item_from_value(
+    item_val: &Value,
+) -> Result<(NewConversationItem, Option<String>), String> {
+    let item_type = item_val
+        .get("type")
+        .and_then(|v| v.as_str())
+        .unwrap_or("message");
+
+    if !SUPPORTED_ITEM_TYPES.contains(&item_type) {
+        return Err(format!(
+            "Unsupported item type '{}'. Supported types: {}",
+            item_type,
+            SUPPORTED_ITEM_TYPES.join(", ")
+        ));
+    }
+
+    let warning = if !IMPLEMENTED_ITEM_TYPES.contains(&item_type) {
+        Some(format!(
+            "Item type '{}' is accepted but not yet implemented. \
+             The item will be stored but may not function as expected.",
+            item_type
+        ))
+    } else {
+        None
+    };
+
+    let role = item_val
+        .get("role")
+        .and_then(|v| v.as_str())
+        .map(String::from);
+
+    if item_type == "message" && role.is_none() {
+        return Err("Message items require 'role' field".to_string());
+    }
+
+    let status = item_val
+        .get("status")
+        .and_then(|v| v.as_str())
+        .map(String::from)
+        .or_else(|| Some("completed".to_string()));
+
+    let content = if item_type == "message" || item_type == "reasoning" {
+        item_val.get("content").cloned().unwrap_or(json!([]))
+    } else {
+        item_val.clone()
+    };
+
+    Ok((
+        NewConversationItem {
+            id: None,
+            response_id: None,
+            item_type: item_type.to_string(),
+            role,
+            content,
+            status,
+        },
+        warning,
+    ))
+}
+
+pub fn conversation_to_json(conversation: &Conversation) -> Value {
+    let mut obj = json!({
+        "id": conversation.id.0,
+        "object": "conversation",
+        "created_at": conversation.created_at.timestamp()
+    });
+
+    if let Some(metadata) = &conversation.metadata {
+        if !metadata.is_empty() {
+            obj["metadata"] = Value::Object(metadata.clone());
+        }
+    }
+
+    obj
+}

third_party/sglang/sgl-model-gateway/src/routers/conversations/mod.rs

@@ -0,0 +1,8 @@
+//! Shared conversation management module.
+//!
+//! This module provides conversation CRUD operations that can be shared
+//! across different router implementations.
+
+mod handlers;
+
+pub use handlers::*;

third_party/sglang/sgl-model-gateway/src/routers/error.rs

@@ -0,0 +1,95 @@
+use axum::{
+    http::{HeaderMap, HeaderValue, StatusCode},
+    response::{IntoResponse, Response},
+    Json,
+};
+use serde::Serialize;
+
+#[derive(Serialize)]
+struct ErrorResponse<'a> {
+    error: ErrorDetail<'a>,
+}
+
+#[derive(Serialize)]
+struct ErrorDetail<'a> {
+    #[serde(rename = "type")]
+    error_type: &'static str,
+    code: &'a str,
+    message: &'a str,
+}
+
+pub const HEADER_X_SMG_ERROR_CODE: &str = "X-SMG-Error-Code";
+
+pub fn internal_error(code: impl Into<String>, message: impl Into<String>) -> Response {
+    create_error(StatusCode::INTERNAL_SERVER_ERROR, code, message)
+}
+
+pub fn bad_request(code: impl Into<String>, message: impl Into<String>) -> Response {
+    create_error(StatusCode::BAD_REQUEST, code, message)
+}
+
+pub fn not_found(code: impl Into<String>, message: impl Into<String>) -> Response {
+    create_error(StatusCode::NOT_FOUND, code, message)
+}
+
+pub fn service_unavailable(code: impl Into<String>, message: impl Into<String>) -> Response {
+    create_error(StatusCode::SERVICE_UNAVAILABLE, code, message)
+}
+
+pub fn failed_dependency(code: impl Into<String>, message: impl Into<String>) -> Response {
+    create_error(StatusCode::FAILED_DEPENDENCY, code, message)
+}
+
+pub fn not_implemented(code: impl Into<String>, message: impl Into<String>) -> Response {
+    create_error(StatusCode::NOT_IMPLEMENTED, code, message)
+}
+
+pub fn bad_gateway(code: impl Into<String>, message: impl Into<String>) -> Response {
+    create_error(StatusCode::BAD_GATEWAY, code, message)
+}
+
+pub fn method_not_allowed(code: impl Into<String>, message: impl Into<String>) -> Response {
+    create_error(StatusCode::METHOD_NOT_ALLOWED, code, message)
+}
+
+pub fn create_error(
+    status: StatusCode,
+    code: impl Into<String>,
+    message: impl Into<String>,
+) -> Response {
+    let code_str = code.into();
+    let message_str = message.into();
+
+    let mut headers = HeaderMap::with_capacity(1);
+    headers.insert(
+        HEADER_X_SMG_ERROR_CODE,
+        HeaderValue::from_str(&code_str).unwrap(),
+    );
+
+    (
+        status,
+        headers,
+        Json(ErrorResponse {
+            error: ErrorDetail {
+                error_type: status_code_to_str(status),
+                code: &code_str,
+                message: &message_str,
+            },
+        }),
+    )
+        .into_response()
+}
+
+fn status_code_to_str(status_code: StatusCode) -> &'static str {
+    status_code
+        .canonical_reason()
+        .unwrap_or("Unknown Status Code")
+}
+
+pub fn extract_error_code_from_response<B>(response: &Response<B>) -> &str {
+    response
+        .headers()
+        .get(HEADER_X_SMG_ERROR_CODE)
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or_default()
+}

third_party/sglang/sgl-model-gateway/src/routers/factory.rs

@@ -0,0 +1,130 @@
+//! Factory for creating router instances
+
+use std::sync::Arc;
+
+use super::{
+    grpc::{pd_router::GrpcPDRouter, router::GrpcRouter},
+    http::{pd_router::PDRouter, router::Router},
+    openai::OpenAIRouter,
+    RouterTrait,
+};
+use crate::{
+    app_context::AppContext,
+    config::{PolicyConfig, RoutingMode},
+    core::ConnectionMode,
+    policies::PolicyFactory,
+};
+
+/// Factory for creating router instances based on configuration
+pub struct RouterFactory;
+
+impl RouterFactory {
+    /// Create a router instance from application context
+    pub async fn create_router(ctx: &Arc<AppContext>) -> Result<Box<dyn RouterTrait>, String> {
+        match ctx.router_config.connection_mode {
+            ConnectionMode::Grpc { .. } => match &ctx.router_config.mode {
+                RoutingMode::Regular { .. } => Self::create_grpc_router(ctx).await,
+                RoutingMode::PrefillDecode {
+                    prefill_policy,
+                    decode_policy,
+                    ..
+                } => {
+                    Self::create_grpc_pd_router(
+                        prefill_policy.as_ref(),
+                        decode_policy.as_ref(),
+                        &ctx.router_config.policy,
+                        ctx,
+                    )
+                    .await
+                }
+                RoutingMode::OpenAI { .. } => {
+                    Err("OpenAI mode requires HTTP connection_mode".to_string())
+                }
+            },
+            ConnectionMode::Http => match &ctx.router_config.mode {
+                RoutingMode::Regular { .. } => Self::create_regular_router(ctx).await,
+                RoutingMode::PrefillDecode {
+                    prefill_policy,
+                    decode_policy,
+                    ..
+                } => {
+                    Self::create_pd_router(
+                        prefill_policy.as_ref(),
+                        decode_policy.as_ref(),
+                        &ctx.router_config.policy,
+                        ctx,
+                    )
+                    .await
+                }
+                RoutingMode::OpenAI { .. } => Self::create_openai_router(ctx).await,
+            },
+        }
+    }
+
+    /// Create a regular router
+    pub async fn create_regular_router(
+        ctx: &Arc<AppContext>,
+    ) -> Result<Box<dyn RouterTrait>, String> {
+        let router = Router::new(ctx).await?;
+
+        Ok(Box::new(router))
+    }
+
+    /// Create a PD router with injected policy
+    pub async fn create_pd_router(
+        prefill_policy_config: Option<&PolicyConfig>,
+        decode_policy_config: Option<&PolicyConfig>,
+        main_policy_config: &PolicyConfig,
+        ctx: &Arc<AppContext>,
+    ) -> Result<Box<dyn RouterTrait>, String> {
+        let prefill_policy =
+            PolicyFactory::create_from_config(prefill_policy_config.unwrap_or(main_policy_config));
+        let decode_policy =
+            PolicyFactory::create_from_config(decode_policy_config.unwrap_or(main_policy_config));
+
+        ctx.policy_registry.set_prefill_policy(prefill_policy);
+        ctx.policy_registry.set_decode_policy(decode_policy);
+
+        let router = PDRouter::new(ctx).await?;
+
+        Ok(Box::new(router))
+    }
+
+    /// Create a gRPC router with injected policy
+    pub async fn create_grpc_router(ctx: &Arc<AppContext>) -> Result<Box<dyn RouterTrait>, String> {
+        let router = GrpcRouter::new(ctx).await?;
+
+        Ok(Box::new(router))
+    }
+
+    /// Create a gRPC PD router with tokenizer and worker configuration
+    pub async fn create_grpc_pd_router(
+        prefill_policy_config: Option<&PolicyConfig>,
+        decode_policy_config: Option<&PolicyConfig>,
+        main_policy_config: &PolicyConfig,
+        ctx: &Arc<AppContext>,
+    ) -> Result<Box<dyn RouterTrait>, String> {
+        let prefill_policy =
+            PolicyFactory::create_from_config(prefill_policy_config.unwrap_or(main_policy_config));
+        let decode_policy =
+            PolicyFactory::create_from_config(decode_policy_config.unwrap_or(main_policy_config));
+
+        ctx.policy_registry.set_prefill_policy(prefill_policy);
+        ctx.policy_registry.set_decode_policy(decode_policy);
+        let router = GrpcPDRouter::new(ctx).await?;
+
+        Ok(Box::new(router))
+    }
+
+    /// Create an OpenAI router
+    ///
+    /// Workers should be registered via the external worker registration workflow
+    /// before using this router. The workflow discovers models from the provided
+    /// endpoints and creates external workers in the registry.
+    pub async fn create_openai_router(
+        ctx: &Arc<AppContext>,
+    ) -> Result<Box<dyn RouterTrait>, String> {
+        let router = OpenAIRouter::new(ctx).await?;
+        Ok(Box::new(router))
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/client.rs

@@ -0,0 +1,208 @@
+//! Unified gRPC client wrapper for SGLang and vLLM backends
+
+use std::sync::Arc;
+
+use smg_grpc_client::{SglangSchedulerClient, VllmEngineClient};
+
+use crate::{
+    observability::otel_trace::OtelTraceInjector,
+    routers::grpc::proto_wrapper::{
+        ProtoEmbedRequest, ProtoEmbedResponse, ProtoGenerateRequest, ProtoStream,
+    },
+};
+
+/// Health check response (common across backends)
+#[derive(Debug, Clone)]
+pub struct HealthCheckResponse {
+    pub healthy: bool,
+    pub message: String,
+}
+
+/// Polymorphic gRPC client that wraps either SGLang or vLLM
+#[derive(Clone)]
+pub enum GrpcClient {
+    Sglang(SglangSchedulerClient),
+    Vllm(VllmEngineClient),
+}
+
+impl GrpcClient {
+    /// Get reference to SGLang client (panics if vLLM)
+    pub fn as_sglang(&self) -> &SglangSchedulerClient {
+        match self {
+            Self::Sglang(client) => client,
+            Self::Vllm(_) => panic!("Expected SGLang client, got vLLM"),
+        }
+    }
+
+    /// Get mutable reference to SGLang client (panics if vLLM)
+    pub fn as_sglang_mut(&mut self) -> &mut SglangSchedulerClient {
+        match self {
+            Self::Sglang(client) => client,
+            Self::Vllm(_) => panic!("Expected SGLang client, got vLLM"),
+        }
+    }
+
+    /// Get reference to vLLM client (panics if SGLang)
+    pub fn as_vllm(&self) -> &VllmEngineClient {
+        match self {
+            Self::Vllm(client) => client,
+            Self::Sglang(_) => panic!("Expected vLLM client, got SGLang"),
+        }
+    }
+
+    /// Get mutable reference to vLLM client (panics if SGLang)
+    pub fn as_vllm_mut(&mut self) -> &mut VllmEngineClient {
+        match self {
+            Self::Vllm(client) => client,
+            Self::Sglang(_) => panic!("Expected vLLM client, got SGLang"),
+        }
+    }
+
+    /// Check if this is a SGLang client
+    pub fn is_sglang(&self) -> bool {
+        matches!(self, Self::Sglang(_))
+    }
+
+    /// Check if this is a vLLM client
+    pub fn is_vllm(&self) -> bool {
+        matches!(self, Self::Vllm(_))
+    }
+
+    /// Connect to gRPC server (runtime-aware)
+    pub async fn connect(
+        url: &str,
+        runtime_type: &str,
+    ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
+        let trace_injector = Arc::new(OtelTraceInjector);
+        match runtime_type {
+            "sglang" => Ok(Self::Sglang(
+                SglangSchedulerClient::connect_with_trace_injector(url, trace_injector).await?,
+            )),
+            "vllm" => Ok(Self::Vllm(
+                VllmEngineClient::connect_with_trace_injector(url, trace_injector).await?,
+            )),
+            _ => Err(format!("Unknown runtime type: {}", runtime_type).into()),
+        }
+    }
+
+    /// Perform health check (dispatches to appropriate backend)
+    pub async fn health_check(
+        &self,
+    ) -> Result<HealthCheckResponse, Box<dyn std::error::Error + Send + Sync>> {
+        match self {
+            Self::Sglang(client) => {
+                let resp = client.health_check().await?;
+                Ok(HealthCheckResponse {
+                    healthy: resp.healthy,
+                    message: resp.message,
+                })
+            }
+            Self::Vllm(client) => {
+                let resp = client.health_check().await?;
+                Ok(HealthCheckResponse {
+                    healthy: resp.healthy,
+                    message: resp.message,
+                })
+            }
+        }
+    }
+
+    /// Get model info (returns enum wrapping backend-specific response)
+    pub async fn get_model_info(
+        &self,
+    ) -> Result<ModelInfo, Box<dyn std::error::Error + Send + Sync>> {
+        match self {
+            Self::Sglang(client) => {
+                let info = client.get_model_info().await?;
+                Ok(ModelInfo::Sglang(Box::new(info)))
+            }
+            Self::Vllm(client) => {
+                let info = client.get_model_info().await?;
+                Ok(ModelInfo::Vllm(info))
+            }
+        }
+    }
+
+    /// Generate streaming response from request
+    ///
+    /// Dispatches to the appropriate backend client and wraps the result in ProtoStream
+    pub async fn generate(
+        &mut self,
+        req: ProtoGenerateRequest,
+    ) -> Result<ProtoStream, Box<dyn std::error::Error + Send + Sync>> {
+        match (self, req) {
+            (Self::Sglang(client), ProtoGenerateRequest::Sglang(boxed_req)) => {
+                let stream = client.generate(*boxed_req).await?;
+                Ok(ProtoStream::Sglang(stream))
+            }
+            (Self::Vllm(client), ProtoGenerateRequest::Vllm(boxed_req)) => {
+                let stream = client.generate(*boxed_req).await?;
+                Ok(ProtoStream::Vllm(stream))
+            }
+            _ => panic!("Mismatched client and request types"),
+        }
+    }
+
+    /// Submit an embedding request
+    pub async fn embed(
+        &mut self,
+        req: ProtoEmbedRequest,
+    ) -> Result<ProtoEmbedResponse, Box<dyn std::error::Error + Send + Sync>> {
+        match (self, req) {
+            (Self::Sglang(client), ProtoEmbedRequest::Sglang(boxed_req)) => {
+                let resp = client.embed(*boxed_req).await?;
+                Ok(ProtoEmbedResponse::Sglang(resp))
+            }
+            _ => panic!("Mismatched client and request types or unsupported embedding backend"),
+        }
+    }
+}
+
+/// Unified ModelInfo wrapper
+pub enum ModelInfo {
+    Sglang(Box<smg_grpc_client::sglang_proto::GetModelInfoResponse>),
+    Vllm(smg_grpc_client::vllm_proto::GetModelInfoResponse),
+}
+
+impl ModelInfo {
+    /// Convert model info to label map for worker metadata
+    pub fn to_labels(&self) -> std::collections::HashMap<String, String> {
+        let mut labels = std::collections::HashMap::new();
+
+        // Serialize to JSON Value (like pydantic's model_dump)
+        let value = match self {
+            ModelInfo::Sglang(info) => serde_json::to_value(info).ok(),
+            ModelInfo::Vllm(info) => serde_json::to_value(info).ok(),
+        };
+
+        // Convert JSON object to HashMap, filtering out empty/zero/false values
+        if let Some(serde_json::Value::Object(obj)) = value {
+            for (key, val) in obj {
+                match val {
+                    // Insert non-empty strings
+                    serde_json::Value::String(s) if !s.is_empty() => {
+                        labels.insert(key, s);
+                    }
+                    // Insert positive numbers
+                    serde_json::Value::Number(n) if n.as_i64().unwrap_or(0) > 0 => {
+                        labels.insert(key, n.to_string());
+                    }
+                    // Insert true booleans
+                    serde_json::Value::Bool(true) => {
+                        labels.insert(key, "true".to_string());
+                    }
+                    // Insert non-empty arrays as JSON strings (for architectures, etc.)
+                    serde_json::Value::Array(arr) if !arr.is_empty() => {
+                        if let Ok(json_str) = serde_json::to_string(&arr) {
+                            labels.insert(key, json_str);
+                        }
+                    }
+                    // Skip empty strings, zeros, false, nulls, empty arrays, objects
+                    _ => {}
+                }
+            }
+        }
+
+        labels
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/mod.rs

@@ -0,0 +1,6 @@
+//! Shared code for both regular and harmony routers
+
+pub(crate) mod response_collection;
+pub(crate) mod response_formatting;
+pub(crate) mod responses;
+pub(crate) mod stages;

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/response_collection.rs

@@ -0,0 +1,98 @@
+//! Shared response collection logic
+//!
+//! This module contains common logic for collecting responses from execution results.
+//! Both regular and harmony processors use these functions to avoid duplication.
+
+use axum::response::Response;
+
+use crate::routers::{
+    error,
+    grpc::{context::ExecutionResult, proto_wrapper::ProtoGenerateComplete, utils},
+};
+
+/// Collect and merge responses from execution result
+///
+/// Handles both Single and Dual (prefill-decode) execution modes.
+/// For Dual mode, merges prefill input_logprobs into decode responses if requested.
+///
+/// # Arguments
+/// * `execution_result` - The execution result containing stream(s)
+/// * `merge_logprobs` - Whether to merge prefill input_logprobs (for chat with logprobs=true)
+///
+/// # Returns
+/// Vector of GenerateComplete responses, one per index (n parameter)
+pub(crate) async fn collect_responses(
+    execution_result: ExecutionResult,
+    merge_logprobs: bool,
+) -> Result<Vec<ProtoGenerateComplete>, Response> {
+    let all_responses = match execution_result {
+        ExecutionResult::Single { mut stream } => {
+            let responses = utils::collect_stream_responses(&mut stream, "Single").await?;
+            stream.mark_completed();
+            responses
+        }
+        ExecutionResult::Dual {
+            mut prefill,
+            decode,
+        } => {
+            // Collect prefill for input_logprobs (don't mark completed yet)
+            let prefill_responses =
+                utils::collect_stream_responses(&mut prefill, "Prefill").await?;
+
+            // Collect decode for actual output (don't mark completed yet)
+            let mut decode_stream = *decode;
+            let mut decode_responses =
+                utils::collect_stream_responses(&mut decode_stream, "Decode").await?;
+
+            // Mark both streams as completed now that both succeeded
+            prefill.mark_completed();
+            decode_stream.mark_completed();
+
+            // Merge prefill input_logprobs if requested
+            if merge_logprobs {
+                merge_prefill_logprobs(&prefill_responses, &mut decode_responses);
+            }
+
+            decode_responses
+        }
+        ExecutionResult::Embedding { .. } => {
+            // Embeddings do not support this path (no generate complete response)
+            return Err(error::internal_error(
+                "invalid_execution_mode",
+                "Embedding result encountered in response collection",
+            ));
+        }
+    };
+
+    if all_responses.is_empty() {
+        return Err(error::internal_error(
+            "no_responses_from_server",
+            "No responses from server",
+        ));
+    }
+
+    Ok(all_responses)
+}
+
+/// Merge prefill input_logprobs into decode responses
+///
+/// Takes input_logprobs from the first prefill response and copies them
+/// into all decode responses. This is used in PD mode when logprobs are requested.
+/// Only works with SGLang (vLLM doesn't support PD mode).
+fn merge_prefill_logprobs(
+    prefill_responses: &[ProtoGenerateComplete],
+    decode_responses: &mut [ProtoGenerateComplete],
+) {
+    // Only SGLang supports PD mode and has input_logprobs
+    if let Some(ProtoGenerateComplete::Sglang(prefill_first)) = prefill_responses.first() {
+        // Use ref to borrow input_logprobs instead of cloning upfront
+        // This avoids one allocation when the Option is Some
+        if let Some(ref prefill_input_logprobs) = prefill_first.input_logprobs {
+            for response in decode_responses.iter_mut() {
+                if let ProtoGenerateComplete::Sglang(decode_resp) = response {
+                    decode_resp.input_logprobs = Some(prefill_input_logprobs.clone());
+                }
+            }
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/response_formatting.rs

@@ -0,0 +1,29 @@
+//! Shared response formatting logic
+//!
+//! This module contains common logic for formatting responses, including:
+//! - Usage calculation from gRPC responses
+//! - ChatCompletionResponse construction
+
+use crate::{protocols::common::Usage, routers::grpc::proto_wrapper::ProtoGenerateComplete};
+
+/// Build usage information from collected gRPC responses
+///
+/// Sums prompt_tokens and completion_tokens across all responses.
+/// Typically used with n>1 parameter where multiple completions are generated.
+///
+/// # Arguments
+/// * `responses` - Vector of GenerateComplete responses from the backend
+///
+/// # Returns
+/// Usage object with aggregated token counts
+pub(crate) fn build_usage(responses: &[ProtoGenerateComplete]) -> Usage {
+    let total_prompt_tokens: u32 = responses.iter().map(|r| r.prompt_tokens() as u32).sum();
+    let total_completion_tokens: u32 = responses.iter().map(|r| r.completion_tokens() as u32).sum();
+
+    Usage {
+        prompt_tokens: total_prompt_tokens,
+        completion_tokens: total_completion_tokens,
+        total_tokens: total_prompt_tokens + total_completion_tokens,
+        completion_tokens_details: None,
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/responses/context.rs

@@ -0,0 +1,60 @@
+//! Shared context for /v1/responses endpoint handlers
+//!
+//! This context is used by both regular and harmony response implementations.
+
+use std::sync::{Arc, RwLock as StdRwLock};
+
+use data_connector::{ConversationItemStorage, ConversationStorage, ResponseStorage};
+use smg_mcp::McpManager;
+
+use crate::routers::grpc::{context::SharedComponents, pipeline::RequestPipeline};
+
+/// Context for /v1/responses endpoint
+///
+/// Used by both regular and harmony implementations.
+/// All fields are Arc/shared references, so cloning this context is cheap.
+#[derive(Clone)]
+pub(crate) struct ResponsesContext {
+    /// Chat pipeline for executing requests
+    pub pipeline: Arc<RequestPipeline>,
+
+    /// Shared components (tokenizer, parsers)
+    pub components: Arc<SharedComponents>,
+
+    /// Response storage backend
+    pub response_storage: Arc<dyn ResponseStorage>,
+
+    /// Conversation storage backend
+    pub conversation_storage: Arc<dyn ConversationStorage>,
+
+    /// Conversation item storage backend
+    pub conversation_item_storage: Arc<dyn ConversationItemStorage>,
+
+    /// MCP manager for tool support
+    pub mcp_manager: Arc<McpManager>,
+
+    /// Server keys for MCP tools requested in this context
+    pub requested_servers: Arc<StdRwLock<Vec<String>>>,
+}
+
+impl ResponsesContext {
+    /// Create a new responses context
+    pub fn new(
+        pipeline: Arc<RequestPipeline>,
+        components: Arc<SharedComponents>,
+        response_storage: Arc<dyn ResponseStorage>,
+        conversation_storage: Arc<dyn ConversationStorage>,
+        conversation_item_storage: Arc<dyn ConversationItemStorage>,
+        mcp_manager: Arc<McpManager>,
+    ) -> Self {
+        Self {
+            pipeline,
+            components,
+            response_storage,
+            conversation_storage,
+            conversation_item_storage,
+            mcp_manager,
+            requested_servers: Arc::new(StdRwLock::new(Vec::new())),
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/responses/handlers.rs

@@ -0,0 +1,74 @@
+//! Shared response handlers for both regular and harmony implementations
+//!
+//! These handlers are used by both pipelines for retrieving and cancelling responses.
+
+use axum::response::{IntoResponse, Response};
+use data_connector::ResponseId;
+
+use super::ResponsesContext;
+use crate::routers::error;
+
+/// Implementation for GET /v1/responses/{response_id}
+///
+/// Retrieves a stored response from the database.
+/// Used by both regular and harmony implementations.
+pub(crate) async fn get_response_impl(ctx: &ResponsesContext, response_id: &str) -> Response {
+    let resp_id = ResponseId::from(response_id);
+
+    // Retrieve response from storage
+    match ctx.response_storage.get_response(&resp_id).await {
+        Ok(Some(stored_response)) => axum::Json(stored_response.raw_response).into_response(),
+        Ok(None) => error::not_found(
+            "response_not_found",
+            format!("Response with id '{}' not found", response_id),
+        ),
+        Err(e) => error::internal_error(
+            "retrieve_response_failed",
+            format!("Failed to retrieve response: {}", e),
+        ),
+    }
+}
+
+/// Implementation for POST /v1/responses/{response_id}/cancel
+///
+/// Background mode is no longer supported, so this endpoint always returns
+/// an error indicating that cancellation is not available.
+pub(crate) async fn cancel_response_impl(ctx: &ResponsesContext, response_id: &str) -> Response {
+    let resp_id = ResponseId::from(response_id);
+
+    // Check if response exists
+    match ctx.response_storage.get_response(&resp_id).await {
+        Ok(Some(stored_response)) => {
+            let current_status = stored_response
+                .raw_response
+                .get("status")
+                .and_then(|v| v.as_str())
+                .unwrap_or("unknown");
+
+            match current_status {
+                "completed" => error::bad_request(
+                    "response_already_completed",
+                    "Cannot cancel completed response",
+                ),
+                "failed" => {
+                    error::bad_request("response_already_failed", "Cannot cancel failed response")
+                }
+                _ => {
+                    // Background mode is no longer supported, so there's nothing to cancel
+                    error::bad_request(
+                        "cancellation_not_supported",
+                        "Background mode is not supported. Synchronous and streaming responses cannot be cancelled.",
+                    )
+                }
+            }
+        }
+        Ok(None) => error::not_found(
+            "response_not_found",
+            format!("Response with id '{}' not found", response_id),
+        ),
+        Err(e) => error::internal_error(
+            "retrieve_response_failed",
+            format!("Failed to retrieve response: {}", e),
+        ),
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/responses/mod.rs

@@ -0,0 +1,11 @@
+//! Shared response functionality used by both regular and harmony implementations
+
+pub(crate) mod context;
+pub(crate) mod handlers;
+pub(crate) mod streaming;
+pub(crate) mod utils;
+
+// Re-export commonly used items
+pub(crate) use context::ResponsesContext;
+pub(crate) use streaming::build_sse_response;
+pub(crate) use utils::{ensure_mcp_connection, persist_response_if_needed};

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/responses/streaming.rs

@@ -0,0 +1,842 @@
+//! Streaming infrastructure for /v1/responses endpoint
+
+use std::collections::HashMap;
+
+use axum::{body::Body, http::StatusCode, response::Response};
+use bytes::Bytes;
+use serde_json::json;
+use smg_mcp as mcp;
+use tokio::sync::mpsc;
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use uuid::Uuid;
+
+use crate::{
+    protocols::{
+        chat::ChatCompletionStreamResponse,
+        common::{Usage, UsageInfo},
+        event_types::{
+            ContentPartEvent, FunctionCallEvent, McpEvent, OutputItemEvent, OutputTextEvent,
+            ResponseEvent,
+        },
+        responses::{
+            ResponseOutputItem, ResponseStatus, ResponsesRequest, ResponsesResponse, ResponsesUsage,
+        },
+    },
+    routers::grpc::harmony::responses::ToolResult,
+};
+
+pub(crate) enum OutputItemType {
+    Message,
+    McpListTools,
+    McpCall,
+    FunctionCall,
+    Reasoning,
+}
+
+/// Status of an output item
+#[derive(Debug, Clone, PartialEq)]
+enum ItemStatus {
+    InProgress,
+    Completed,
+}
+
+/// State tracking for a single output item
+#[derive(Debug, Clone)]
+struct OutputItemState {
+    output_index: usize,
+    status: ItemStatus,
+    item_data: Option<serde_json::Value>,
+}
+
+/// OpenAI-compatible event emitter for /v1/responses streaming
+///
+/// Manages state and sequence numbers to emit proper event types:
+/// - response.created
+/// - response.in_progress
+/// - response.output_item.added
+/// - response.content_part.added
+/// - response.output_text.delta (multiple)
+/// - response.output_text.done
+/// - response.content_part.done
+/// - response.output_item.done
+/// - response.completed
+/// - response.mcp_list_tools.in_progress
+/// - response.mcp_list_tools.completed
+/// - response.mcp_call.in_progress
+/// - response.mcp_call_arguments.delta
+/// - response.mcp_call_arguments.done
+/// - response.mcp_call.completed
+/// - response.mcp_call.failed
+pub(crate) struct ResponseStreamEventEmitter {
+    sequence_number: u64,
+    pub response_id: String,
+    model: String,
+    created_at: u64,
+    message_id: String,
+    accumulated_text: String,
+    has_emitted_created: bool,
+    has_emitted_in_progress: bool,
+    has_emitted_output_item_added: bool,
+    has_emitted_content_part_added: bool,
+    // MCP call tracking
+    mcp_call_accumulated_args: HashMap<String, String>,
+    pub(crate) mcp_server_label: Option<String>, // Server label for MCP tools
+    // Output item tracking
+    output_items: Vec<OutputItemState>,
+    next_output_index: usize,
+    current_message_output_index: Option<usize>, // Tracks output_index of current message
+    current_item_id: Option<String>,             // Tracks item_id of current item
+    original_request: Option<ResponsesRequest>,
+}
+
+impl ResponseStreamEventEmitter {
+    pub fn new(response_id: String, model: String, created_at: u64) -> Self {
+        let message_id = format!("msg_{}", Uuid::new_v4());
+
+        Self {
+            sequence_number: 0,
+            response_id,
+            model,
+            created_at,
+            message_id,
+            accumulated_text: String::new(),
+            has_emitted_created: false,
+            has_emitted_in_progress: false,
+            has_emitted_output_item_added: false,
+            has_emitted_content_part_added: false,
+            mcp_call_accumulated_args: HashMap::new(),
+            mcp_server_label: None,
+            output_items: Vec::new(),
+            next_output_index: 0,
+            current_message_output_index: None,
+            current_item_id: None,
+            original_request: None,
+        }
+    }
+
+    /// Set the original request for including all fields in response.completed
+    pub fn set_original_request(&mut self, request: ResponsesRequest) {
+        self.original_request = Some(request);
+    }
+
+    /// Set the MCP server label for MCP tool calls
+    pub fn set_mcp_server_label(&mut self, server_label: String) {
+        self.mcp_server_label = Some(server_label);
+    }
+
+    /// Update mcp_call output items with tool execution results
+    ///
+    /// After MCP tools are executed, this updates the stored output items
+    /// to include the output field from the tool results.
+    pub(crate) fn update_mcp_call_outputs(&mut self, tool_results: &[ToolResult]) {
+        for tool_result in tool_results {
+            // Find the output item with matching call_id
+            for item_state in self.output_items.iter_mut() {
+                if let Some(ref mut item_data) = item_state.item_data {
+                    // Check if this is an mcp_call item with matching call_id
+                    if item_data.get("type").and_then(|t| t.as_str()) == Some("mcp_call")
+                        && item_data.get("call_id").and_then(|c| c.as_str())
+                            == Some(&tool_result.call_id)
+                    {
+                        // Add output field
+                        let output_str = serde_json::to_string(&tool_result.output)
+                            .unwrap_or_else(|_| "{}".to_string());
+                        item_data["output"] = json!(output_str);
+
+                        // Update status based on success
+                        if tool_result.is_error {
+                            item_data["status"] = json!("failed");
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    fn next_sequence(&mut self) -> u64 {
+        let seq = self.sequence_number;
+        self.sequence_number += 1;
+        seq
+    }
+
+    pub fn emit_created(&mut self) -> serde_json::Value {
+        self.has_emitted_created = true;
+        json!({
+            "type": ResponseEvent::CREATED,
+            "sequence_number": self.next_sequence(),
+            "response": {
+                "id": self.response_id,
+                "object": "response",
+                "created_at": self.created_at,
+                "status": "in_progress",
+                "model": self.model,
+                "output": []
+            }
+        })
+    }
+
+    pub fn emit_in_progress(&mut self) -> serde_json::Value {
+        self.has_emitted_in_progress = true;
+        json!({
+            "type": ResponseEvent::IN_PROGRESS,
+            "sequence_number": self.next_sequence(),
+            "response": {
+                "id": self.response_id,
+                "object": "response",
+                "status": "in_progress"
+            }
+        })
+    }
+
+    pub fn emit_content_part_added(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+        content_index: usize,
+    ) -> serde_json::Value {
+        self.has_emitted_content_part_added = true;
+        json!({
+            "type": ContentPartEvent::ADDED,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "content_index": content_index,
+            "part": {
+                "type": "text",
+                "text": ""
+            }
+        })
+    }
+
+    pub fn emit_text_delta(
+        &mut self,
+        delta: &str,
+        output_index: usize,
+        item_id: &str,
+        content_index: usize,
+    ) -> serde_json::Value {
+        self.accumulated_text.push_str(delta);
+        json!({
+            "type": OutputTextEvent::DELTA,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "content_index": content_index,
+            "delta": delta
+        })
+    }
+
+    pub fn emit_text_done(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+        content_index: usize,
+    ) -> serde_json::Value {
+        json!({
+            "type": OutputTextEvent::DONE,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "content_index": content_index,
+            "text": self.accumulated_text.clone()
+        })
+    }
+
+    pub fn emit_content_part_done(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+        content_index: usize,
+    ) -> serde_json::Value {
+        json!({
+            "type": ContentPartEvent::DONE,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "content_index": content_index,
+            "part": {
+                "type": "text",
+                "text": self.accumulated_text.clone()
+            }
+        })
+    }
+
+    pub fn emit_completed(&mut self, usage: Option<&serde_json::Value>) -> serde_json::Value {
+        // Build output array from tracked items
+        let output: Vec<serde_json::Value> = self
+            .output_items
+            .iter()
+            .filter_map(|item| {
+                if item.status == ItemStatus::Completed {
+                    item.item_data.clone()
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        // If no items were tracked (legacy path), fall back to generic message
+        let output = if output.is_empty() {
+            vec![json!({
+                "id": self.message_id.clone(),
+                "type": "message",
+                "role": "assistant",
+                "content": [{
+                    "type": "text",
+                    "text": self.accumulated_text.clone()
+                }]
+            })]
+        } else {
+            output
+        };
+
+        // Build base response object
+        let mut response_obj = json!({
+            "id": self.response_id,
+            "object": "response",
+            "created_at": self.created_at,
+            "status": "completed",
+            "model": self.model,
+            "output": output
+        });
+
+        // Add usage if provided
+        if let Some(usage_val) = usage {
+            response_obj["usage"] = usage_val.clone();
+        }
+
+        // Add all original request fields if available
+        if let Some(ref req) = self.original_request {
+            Self::add_optional_field(&mut response_obj, "instructions", &req.instructions);
+            Self::add_optional_field(
+                &mut response_obj,
+                "max_output_tokens",
+                &req.max_output_tokens,
+            );
+            Self::add_optional_field(&mut response_obj, "max_tool_calls", &req.max_tool_calls);
+            Self::add_optional_field(
+                &mut response_obj,
+                "previous_response_id",
+                &req.previous_response_id,
+            );
+            Self::add_optional_field(&mut response_obj, "reasoning", &req.reasoning);
+            Self::add_optional_field(&mut response_obj, "temperature", &req.temperature);
+            Self::add_optional_field(&mut response_obj, "top_p", &req.top_p);
+            Self::add_optional_field(&mut response_obj, "truncation", &req.truncation);
+            Self::add_optional_field(&mut response_obj, "user", &req.user);
+
+            response_obj["parallel_tool_calls"] = json!(req.parallel_tool_calls.unwrap_or(true));
+            response_obj["store"] = json!(req.store.unwrap_or(true));
+            response_obj["tools"] = json!(req.tools.as_ref().unwrap_or(&vec![]));
+            response_obj["metadata"] = json!(req.metadata.as_ref().unwrap_or(&Default::default()));
+
+            // tool_choice: serialize if present, otherwise use "auto"
+            if let Some(ref tc) = req.tool_choice {
+                response_obj["tool_choice"] = json!(tc);
+            } else {
+                response_obj["tool_choice"] = json!("auto");
+            }
+        }
+
+        json!({
+            "type": ResponseEvent::COMPLETED,
+            "sequence_number": self.next_sequence(),
+            "response": response_obj
+        })
+    }
+
+    /// Helper to add optional fields to JSON object
+    fn add_optional_field<T: serde::Serialize>(
+        obj: &mut serde_json::Value,
+        key: &str,
+        value: &Option<T>,
+    ) {
+        if let Some(val) = value {
+            obj[key] = json!(val);
+        }
+    }
+
+    // ========================================================================
+    // MCP Event Emission Methods
+    // ========================================================================
+
+    pub fn emit_mcp_list_tools_in_progress(&mut self, output_index: usize) -> serde_json::Value {
+        json!({
+            "type": McpEvent::LIST_TOOLS_IN_PROGRESS,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index
+        })
+    }
+
+    pub fn emit_mcp_list_tools_completed(
+        &mut self,
+        output_index: usize,
+        tools: &[mcp::Tool],
+    ) -> serde_json::Value {
+        let tool_items: Vec<_> = tools
+            .iter()
+            .map(|t| {
+                json!({
+                    "name": &t.name,
+                    "description": &t.description,
+                    "input_schema": t.input_schema.clone()
+                })
+            })
+            .collect();
+
+        json!({
+            "type": McpEvent::LIST_TOOLS_COMPLETED,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "tools": tool_items
+        })
+    }
+
+    pub fn emit_mcp_call_in_progress(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+    ) -> serde_json::Value {
+        json!({
+            "type": McpEvent::CALL_IN_PROGRESS,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id
+        })
+    }
+
+    pub fn emit_mcp_call_arguments_delta(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+        delta: &str,
+    ) -> serde_json::Value {
+        // Accumulate arguments for this call
+        self.mcp_call_accumulated_args
+            .entry(item_id.to_string())
+            .or_default()
+            .push_str(delta);
+
+        json!({
+            "type": McpEvent::CALL_ARGUMENTS_DELTA,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "delta": delta
+        })
+    }
+
+    pub fn emit_mcp_call_arguments_done(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+        arguments: &str,
+    ) -> serde_json::Value {
+        json!({
+            "type": McpEvent::CALL_ARGUMENTS_DONE,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "arguments": arguments
+        })
+    }
+
+    pub fn emit_mcp_call_completed(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+    ) -> serde_json::Value {
+        json!({
+            "type": McpEvent::CALL_COMPLETED,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id
+        })
+    }
+
+    pub fn emit_mcp_call_failed(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+        error: &str,
+    ) -> serde_json::Value {
+        json!({
+            "type": McpEvent::CALL_FAILED,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "error": error
+        })
+    }
+
+    // ========================================================================
+    // Function Call Event Emission Methods
+    // ========================================================================
+
+    pub fn emit_function_call_arguments_delta(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+        delta: &str,
+    ) -> serde_json::Value {
+        json!({
+            "type": FunctionCallEvent::ARGUMENTS_DELTA,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "delta": delta
+        })
+    }
+
+    pub fn emit_function_call_arguments_done(
+        &mut self,
+        output_index: usize,
+        item_id: &str,
+        arguments: &str,
+    ) -> serde_json::Value {
+        json!({
+            "type": FunctionCallEvent::ARGUMENTS_DONE,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item_id": item_id,
+            "arguments": arguments
+        })
+    }
+
+    // ========================================================================
+    // Output Item Wrapper Events
+    // ========================================================================
+
+    /// Emit response.output_item.added event
+    pub fn emit_output_item_added(
+        &mut self,
+        output_index: usize,
+        item: &serde_json::Value,
+    ) -> serde_json::Value {
+        json!({
+            "type": OutputItemEvent::ADDED,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item": item
+        })
+    }
+
+    /// Emit response.output_item.done event
+    pub fn emit_output_item_done(
+        &mut self,
+        output_index: usize,
+        item: &serde_json::Value,
+    ) -> serde_json::Value {
+        // Store the item data for later use in emit_completed
+        self.store_output_item_data(output_index, item.clone());
+
+        json!({
+            "type": OutputItemEvent::DONE,
+            "sequence_number": self.next_sequence(),
+            "output_index": output_index,
+            "item": item
+        })
+    }
+
+    /// Generate unique ID for item type
+    fn generate_item_id(prefix: &str) -> String {
+        format!("{}_{}", prefix, Uuid::new_v4().to_string().replace("-", ""))
+    }
+
+    /// Allocate next output index and track item
+    pub fn allocate_output_index(&mut self, item_type: OutputItemType) -> (usize, String) {
+        let index = self.next_output_index;
+        self.next_output_index += 1;
+
+        let id_prefix = match &item_type {
+            OutputItemType::McpListTools => "mcpl",
+            OutputItemType::McpCall => "mcp",
+            OutputItemType::FunctionCall => "fc",
+            OutputItemType::Message => "msg",
+            OutputItemType::Reasoning => "rs",
+        };
+
+        let id = Self::generate_item_id(id_prefix);
+
+        self.output_items.push(OutputItemState {
+            output_index: index,
+            status: ItemStatus::InProgress,
+            item_data: None,
+        });
+
+        (index, id)
+    }
+
+    /// Mark output item as completed and store its data
+    pub fn complete_output_item(&mut self, output_index: usize) {
+        if let Some(item) = self
+            .output_items
+            .iter_mut()
+            .find(|i| i.output_index == output_index)
+        {
+            item.status = ItemStatus::Completed;
+        }
+    }
+
+    /// Store output item data when emitting output_item.done
+    pub fn store_output_item_data(&mut self, output_index: usize, item_data: serde_json::Value) {
+        if let Some(item) = self
+            .output_items
+            .iter_mut()
+            .find(|i| i.output_index == output_index)
+        {
+            item.item_data = Some(item_data);
+        }
+    }
+
+    /// Finalize and return the complete ResponsesResponse
+    ///
+    /// This constructs the final ResponsesResponse from all accumulated output items
+    /// for persistence. Should be called after streaming is complete.
+    pub fn finalize(&self, usage: Option<Usage>) -> ResponsesResponse {
+        // Build output array from tracked items
+        let output: Vec<ResponseOutputItem> = self
+            .output_items
+            .iter()
+            .filter_map(|item| {
+                item.item_data
+                    .as_ref()
+                    .and_then(|data| serde_json::from_value(data.clone()).ok())
+            })
+            .collect();
+
+        // Convert Usage to ResponsesUsage
+        let responses_usage = usage.map(|u| {
+            let usage_info = UsageInfo {
+                prompt_tokens: u.prompt_tokens,
+                completion_tokens: u.completion_tokens,
+                total_tokens: u.total_tokens,
+                reasoning_tokens: u
+                    .completion_tokens_details
+                    .as_ref()
+                    .and_then(|d| d.reasoning_tokens),
+                prompt_tokens_details: None,
+            };
+            ResponsesUsage::Classic(usage_info)
+        });
+
+        // Build response using builder
+        ResponsesResponse::builder(&self.response_id, &self.model)
+            .created_at(self.created_at as i64)
+            .status(ResponseStatus::Completed)
+            .output(output)
+            .maybe_copy_from_request(self.original_request.as_ref())
+            .maybe_usage(responses_usage)
+            .build()
+    }
+
+    /// Emit reasoning item wrapper events (added + done)
+    ///
+    /// Reasoning items in OpenAI format are simple placeholders emitted between tool iterations.
+    /// They don't have streaming content - just wrapper events with empty/null content.
+    pub fn emit_reasoning_item(
+        &mut self,
+        tx: &mpsc::UnboundedSender<Result<Bytes, std::io::Error>>,
+        reasoning_content: Option<String>,
+    ) -> Result<(), String> {
+        // Allocate output index and generate ID
+        let (output_index, item_id) = self.allocate_output_index(OutputItemType::Reasoning);
+
+        // Build reasoning item structure
+        let item = json!({
+            "id": item_id,
+            "type": "reasoning",
+            "summary": [],
+            "content": reasoning_content,
+            "encrypted_content": null,
+            "status": null
+        });
+
+        // Emit output_item.added
+        let added_event = self.emit_output_item_added(output_index, &item);
+        self.send_event(&added_event, tx)?;
+
+        // Immediately emit output_item.done (no streaming for reasoning)
+        let done_event = self.emit_output_item_done(output_index, &item);
+        self.send_event(&done_event, tx)?;
+
+        // Mark as completed
+        self.complete_output_item(output_index);
+
+        Ok(())
+    }
+
+    /// Process a chunk and emit appropriate events
+    pub fn process_chunk(
+        &mut self,
+        chunk: &ChatCompletionStreamResponse,
+        tx: &mpsc::UnboundedSender<Result<Bytes, std::io::Error>>,
+    ) -> Result<(), String> {
+        // Process content if present
+        if let Some(choice) = chunk.choices.first() {
+            if let Some(content) = &choice.delta.content {
+                if !content.is_empty() {
+                    // Allocate output_index and item_id for this message item (once per message)
+                    if self.current_item_id.is_none() {
+                        let (output_index, item_id) =
+                            self.allocate_output_index(OutputItemType::Message);
+
+                        // Build message item structure
+                        let item = json!({
+                            "id": item_id,
+                            "type": "message",
+                            "role": "assistant",
+                            "content": []
+                        });
+
+                        // Emit output_item.added
+                        let event = self.emit_output_item_added(output_index, &item);
+                        self.send_event(&event, tx)?;
+                        self.has_emitted_output_item_added = true;
+
+                        // Store for subsequent events
+                        self.current_item_id = Some(item_id);
+                        self.current_message_output_index = Some(output_index);
+                    }
+
+                    let output_index = self.current_message_output_index.unwrap();
+                    let item_id = self.current_item_id.clone().unwrap(); // Clone to avoid borrow checker issues
+                    let content_index = 0; // Single content part for now
+
+                    // Emit content_part.added before first delta
+                    if !self.has_emitted_content_part_added {
+                        let event =
+                            self.emit_content_part_added(output_index, &item_id, content_index);
+                        self.send_event(&event, tx)?;
+                        self.has_emitted_content_part_added = true;
+                    }
+
+                    // Emit text delta
+                    let event =
+                        self.emit_text_delta(content, output_index, &item_id, content_index);
+                    self.send_event(&event, tx)?;
+                }
+            }
+
+            // Check for finish_reason to emit completion events
+            if let Some(reason) = &choice.finish_reason {
+                if reason == "stop" || reason == "length" {
+                    let output_index = self.current_message_output_index.unwrap();
+                    let item_id = self.current_item_id.clone().unwrap(); // Clone to avoid borrow checker issues
+                    let content_index = 0;
+
+                    // Emit closing events
+                    if self.has_emitted_content_part_added {
+                        let event = self.emit_text_done(output_index, &item_id, content_index);
+                        self.send_event(&event, tx)?;
+                        let event =
+                            self.emit_content_part_done(output_index, &item_id, content_index);
+                        self.send_event(&event, tx)?;
+                    }
+
+                    if self.has_emitted_output_item_added {
+                        // Build complete message item for output_item.done
+                        let item = json!({
+                            "id": item_id,
+                            "type": "message",
+                            "role": "assistant",
+                            "content": [{
+                                "type": "text",
+                                "text": self.accumulated_text.clone()
+                            }]
+                        });
+                        let event = self.emit_output_item_done(output_index, &item);
+                        self.send_event(&event, tx)?;
+                    }
+
+                    // Mark item as completed
+                    self.complete_output_item(output_index);
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn send_event(
+        &self,
+        event: &serde_json::Value,
+        tx: &mpsc::UnboundedSender<Result<Bytes, std::io::Error>>,
+    ) -> Result<(), String> {
+        let event_json = serde_json::to_string(event)
+            .map_err(|e| format!("Failed to serialize event: {}", e))?;
+
+        // Extract event type from the JSON for SSE event field
+        let event_type = event
+            .get("type")
+            .and_then(|v| v.as_str())
+            .unwrap_or("message");
+
+        // Format as SSE with event: field
+        let sse_message = format!("event: {}\ndata: {}\n\n", event_type, event_json);
+
+        if tx.send(Ok(Bytes::from(sse_message))).is_err() {
+            return Err("Client disconnected".to_string());
+        }
+
+        Ok(())
+    }
+
+    /// Send event and log any errors (typically client disconnect)
+    ///
+    /// This is a convenience method for streaming scenarios where client
+    /// disconnection is expected and should be logged but not fail the operation.
+    /// Returns true if sent successfully, false if client disconnected.
+    pub fn send_event_best_effort(
+        &self,
+        event: &serde_json::Value,
+        tx: &mpsc::UnboundedSender<Result<Bytes, std::io::Error>>,
+    ) -> bool {
+        match self.send_event(event, tx) {
+            Ok(()) => true,
+            Err(e) => {
+                tracing::debug!("Failed to send event (likely client disconnect): {}", e);
+                false
+            }
+        }
+    }
+
+    /// Emit an error event
+    ///
+    /// Creates and sends an error event with the given error message.
+    /// Uses OpenAI's error event format.
+    /// Use this for terminal errors that should abort the streaming response.
+    pub fn emit_error(
+        &mut self,
+        error_msg: &str,
+        error_code: Option<&str>,
+        tx: &mpsc::UnboundedSender<Result<Bytes, std::io::Error>>,
+    ) {
+        let event = json!({
+            "type": "error",
+            "code": error_code.unwrap_or("internal_error"),
+            "message": error_msg,
+            "param": null,
+            "sequence_number": self.next_sequence()
+        });
+        let sse_data = format!("data: {}\n\n", serde_json::to_string(&event).unwrap());
+        let _ = tx.send(Ok(Bytes::from(sse_data)));
+    }
+}
+
+/// Build a Server-Sent Events (SSE) response
+///
+/// Creates a Response with proper SSE headers and streaming body.
+pub(crate) fn build_sse_response(
+    rx: mpsc::UnboundedReceiver<Result<Bytes, std::io::Error>>,
+) -> Response {
+    let stream = UnboundedReceiverStream::new(rx);
+    Response::builder()
+        .status(StatusCode::OK)
+        .header("Content-Type", "text/event-stream")
+        .header("Cache-Control", "no-cache")
+        .header("Connection", "keep-alive")
+        .body(Body::from_stream(stream))
+        .unwrap()
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/responses/utils.rs

@@ -0,0 +1,156 @@
+//! Utility functions for /v1/responses endpoint
+
+use std::sync::Arc;
+
+use axum::response::Response;
+use data_connector::{ConversationItemStorage, ConversationStorage, ResponseStorage};
+use serde_json::to_value;
+use smg_mcp::McpManager;
+use tracing::{debug, error, warn};
+
+use crate::{
+    core::WorkerRegistry,
+    protocols::{
+        common::Tool,
+        responses::{ResponseTool, ResponseToolType, ResponsesRequest, ResponsesResponse},
+    },
+    routers::{
+        error, mcp_utils::ensure_request_mcp_client, persistence_utils::persist_conversation_items,
+    },
+};
+
+/// Ensure MCP connection succeeds if MCP tools are declared
+///
+/// Checks if request declares MCP tools, and if so, validates that
+/// the MCP clients can be created and connected.
+/// Returns Ok((has_mcp_tools, server_keys)) on success.
+pub(crate) async fn ensure_mcp_connection(
+    mcp_manager: &Arc<McpManager>,
+    tools: Option<&[ResponseTool]>,
+) -> Result<(bool, Vec<String>), Response> {
+    let has_mcp_tools = tools
+        .map(|t| {
+            t.iter()
+                .any(|tool| matches!(tool.r#type, ResponseToolType::Mcp))
+        })
+        .unwrap_or(false);
+
+    if has_mcp_tools {
+        if let Some(tools) = tools {
+            match ensure_request_mcp_client(mcp_manager, tools).await {
+                Some((_manager, server_keys)) => {
+                    return Ok((true, server_keys));
+                }
+                None => {
+                    error!(
+                        function = "ensure_mcp_connection",
+                        "Failed to connect to MCP servers"
+                    );
+                    return Err(error::failed_dependency(
+                        "connect_mcp_server_failed",
+                        "Failed to connect to MCP servers. Check server_url and authorization.",
+                    ));
+                }
+            }
+        }
+    }
+
+    Ok((false, Vec::new()))
+}
+
+/// Validate that workers are available for the requested model
+pub(crate) fn validate_worker_availability(
+    worker_registry: &Arc<WorkerRegistry>,
+    model: &str,
+) -> Option<Response> {
+    let available_models = worker_registry.get_models();
+
+    if !available_models.contains(&model.to_string()) {
+        return Some(error::service_unavailable(
+            "no_available_workers",
+            format!(
+                "No workers available for model '{}'. Available models: {}",
+                model,
+                available_models.join(", ")
+            ),
+        ));
+    }
+
+    None
+}
+
+/// Extract function tools (and optionally MCP tools) from ResponseTools
+///
+/// This utility consolidates the logic for extracting tools with schemas from ResponseTools.
+/// It's used by both Harmony and Regular routers for different purposes:
+///
+/// - **Harmony router**: Extracts both Function and MCP tools (with `include_mcp: true`)
+///   because MCP schemas are populated by convert_mcp_tools_to_response_tools() before the
+///   pipeline runs. These tools are used to generate structural constraints in the
+///   Harmony preparation stage.
+///
+/// - **Regular router**: Extracts only Function tools (with `include_mcp: false`) during
+///   the initial conversion from ResponsesRequest to ChatCompletionRequest. MCP tools
+///   are merged later by the tool loop before being sent to the chat pipeline, where
+///   tool_choice constraints are generated for ALL tools (function + MCP combined).
+pub(crate) fn extract_tools_from_response_tools(
+    response_tools: Option<&[ResponseTool]>,
+    include_mcp: bool,
+) -> Vec<Tool> {
+    let Some(tools) = response_tools else {
+        return Vec::new();
+    };
+
+    tools
+        .iter()
+        .filter_map(|rt| {
+            match rt.r#type {
+                // Function tools: Schema in request
+                ResponseToolType::Function => rt.function.as_ref().map(|f| Tool {
+                    tool_type: "function".to_string(),
+                    function: f.clone(),
+                }),
+                // MCP tools: Schema populated by convert_mcp_tools_to_response_tools()
+                // Only include if requested (Harmony case)
+                ResponseToolType::Mcp if include_mcp => rt.function.as_ref().map(|f| Tool {
+                    tool_type: "function".to_string(),
+                    function: f.clone(),
+                }),
+                // Hosted tools: No schema available, skip
+                _ => None,
+            }
+        })
+        .collect()
+}
+
+/// Persist response to storage if store=true
+///
+/// Common helper function to avoid duplication across sync and streaming paths
+/// in both harmony and regular responses implementations.
+pub(crate) async fn persist_response_if_needed(
+    conversation_storage: Arc<dyn ConversationStorage>,
+    conversation_item_storage: Arc<dyn ConversationItemStorage>,
+    response_storage: Arc<dyn ResponseStorage>,
+    response: &ResponsesResponse,
+    original_request: &ResponsesRequest,
+) {
+    if !original_request.store.unwrap_or(true) {
+        return;
+    }
+
+    if let Ok(response_json) = to_value(response) {
+        if let Err(e) = persist_conversation_items(
+            conversation_storage,
+            conversation_item_storage,
+            response_storage,
+            &response_json,
+            original_request,
+        )
+        .await
+        {
+            warn!("Failed to persist response: {}", e);
+        } else {
+            debug!("Persisted response: {}", response.id);
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/stages/client_acquisition.rs

@@ -0,0 +1,69 @@
+//! Client acquisition stage: Get gRPC clients from selected workers
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use super::PipelineStage;
+use crate::routers::{
+    error,
+    grpc::{
+        context::{ClientSelection, RequestContext, WorkerSelection},
+        utils,
+    },
+};
+
+/// Client acquisition stage: Get gRPC clients from selected workers
+pub(crate) struct ClientAcquisitionStage;
+
+#[async_trait]
+impl PipelineStage for ClientAcquisitionStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let workers = ctx.state.workers.as_ref().ok_or_else(|| {
+            error!(
+                function = "ClientAcquisitionStage::execute",
+                "Worker selection stage not completed"
+            );
+            error::internal_error(
+                "worker_selection_not_completed",
+                "Worker selection not completed",
+            )
+        })?;
+
+        let clients = match workers {
+            WorkerSelection::Single { worker } => {
+                let client = utils::get_grpc_client_from_worker(worker).await?;
+                ClientSelection::Single { client }
+            }
+            WorkerSelection::Dual { prefill, decode } => {
+                let prefill_client = utils::get_grpc_client_from_worker(prefill).await?;
+                let decode_client = utils::get_grpc_client_from_worker(decode).await?;
+
+                // vLLM does not support dual (PD disaggregated) mode
+                if prefill_client.is_vllm() || decode_client.is_vllm() {
+                    error!(
+                        function = "ClientAcquisitionStage::execute",
+                        "vLLM backend does not support dual (PD disaggregated) mode"
+                    );
+                    return Err(error::bad_request(
+                        "vllm_pd_mode_not_supported",
+                        "vLLM backend does not support prefill/decode disaggregated mode. \
+                         Please use runtime_type: sglang for PD mode, or use a regular (non-PD) worker configuration."
+                    ));
+                }
+
+                ClientSelection::Dual {
+                    prefill: prefill_client,
+                    decode: decode_client,
+                }
+            }
+        };
+
+        ctx.state.clients = Some(clients);
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "ClientAcquisition"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/stages/dispatch_metadata.rs

@@ -0,0 +1,77 @@
+//! Dispatch metadata stage: Prepare metadata for dispatch
+
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use super::PipelineStage;
+use crate::{
+    core::UNKNOWN_MODEL_ID,
+    routers::{
+        error,
+        grpc::context::{DispatchMetadata, RequestContext, RequestType, WorkerSelection},
+    },
+};
+
+/// Dispatch metadata stage: Prepare metadata for dispatch
+pub(crate) struct DispatchMetadataStage;
+
+#[async_trait]
+impl PipelineStage for DispatchMetadataStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let proto_request = ctx.state.proto_request.as_ref().ok_or_else(|| {
+            error!(
+                function = "DispatchMetadataStage::execute",
+                "Proto request not built"
+            );
+            error::internal_error("proto_request_not_built", "Proto request not built")
+        })?;
+
+        let request_id = proto_request.request_id().to_string();
+        let model = match &ctx.input.request_type {
+            RequestType::Chat(req) => req.model.clone(),
+            RequestType::Generate(_req) => {
+                // Generate requests don't have a model field
+                // Use model_id from input or UNKNOWN_MODEL_ID
+                ctx.input
+                    .model_id
+                    .clone()
+                    .unwrap_or_else(|| UNKNOWN_MODEL_ID.to_string())
+            }
+            RequestType::Responses(req) => req.model.clone(),
+            RequestType::Embedding(req) => req.model.clone(),
+            RequestType::Classify(req) => req.model.clone(),
+        };
+
+        let weight_version = ctx
+            .state
+            .workers
+            .as_ref()
+            .map(|w| match w {
+                WorkerSelection::Single { worker } => worker,
+                WorkerSelection::Dual { decode, .. } => decode,
+            })
+            .and_then(|w| w.metadata().labels.get("weight_version").cloned())
+            .unwrap_or_else(|| "default".to_string());
+
+        let created = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs();
+
+        ctx.state.dispatch = Some(DispatchMetadata {
+            request_id,
+            model,
+            created,
+            weight_version: Some(weight_version),
+        });
+
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "DispatchMetadata"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/stages/helpers.rs

@@ -0,0 +1,41 @@
+//! Common helper functions shared across stages
+
+use std::sync::Arc;
+
+use rand::Rng;
+use smg_grpc_client::sglang_proto::DisaggregatedParams;
+use tracing::debug;
+
+use crate::{core::Worker, routers::grpc::proto_wrapper::ProtoGenerateRequest};
+
+/// Inject PD bootstrap metadata into a gRPC request
+///
+/// Used by both chat and generate request building stages when in PD mode.
+/// Only SGLang supports PD (prefill/decode) disaggregated mode.
+pub(crate) fn inject_bootstrap_metadata(
+    request: &mut ProtoGenerateRequest,
+    prefill_worker: &Arc<dyn Worker>,
+) {
+    let hostname = prefill_worker.bootstrap_host();
+    let bootstrap_port = prefill_worker.bootstrap_port().unwrap_or(8998);
+
+    // Generate room ID for bootstrap
+    let room_id = rand::rng().random_range(0..i32::MAX);
+
+    // Create DisaggregatedParams
+    let disagg_params = DisaggregatedParams {
+        bootstrap_host: hostname.to_string(),
+        bootstrap_port: bootstrap_port as i32,
+        bootstrap_room: room_id,
+    };
+
+    // Inject metadata directly into SGLang request
+    // (vLLM doesn't support PD mode, so this will panic if called with vLLM)
+    let sglang_request = request.as_sglang_mut();
+    sglang_request.disaggregated_params = Some(disagg_params);
+
+    debug!(
+        "Injected bootstrap metadata: host={}, port={}, room={}",
+        hostname, bootstrap_port, room_id
+    );
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/stages/mod.rs

@@ -0,0 +1,39 @@
+//! Common pipeline stages shared across all endpoints and model types
+//!
+//! These stages are endpoint-agnostic and model-agnostic:
+//! - Worker selection
+//! - Client acquisition
+//! - Dispatch metadata generation
+//! - Request execution
+
+use async_trait::async_trait;
+use axum::response::Response;
+
+use crate::routers::grpc::context::RequestContext;
+
+/// Trait for pipeline stages that process requests
+#[async_trait]
+pub trait PipelineStage: Send + Sync {
+    /// Execute this stage, mutating the context
+    ///
+    /// Returns:
+    /// - `Ok(None)` - Continue to next stage
+    /// - `Ok(Some(response))` - Pipeline complete, return this response (e.g., streaming)
+    /// - `Err(response)` - Error occurred, return this error response
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response>;
+
+    /// Stage name for logging
+    fn name(&self) -> &'static str;
+}
+
+mod client_acquisition;
+mod dispatch_metadata;
+pub(crate) mod helpers;
+mod request_execution;
+mod worker_selection;
+
+// Export stage implementations
+pub(crate) use client_acquisition::ClientAcquisitionStage;
+pub(crate) use dispatch_metadata::DispatchMetadataStage;
+pub(crate) use request_execution::{ExecutionMode, RequestExecutionStage};
+pub(crate) use worker_selection::{WorkerSelectionMode, WorkerSelectionStage};

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/stages/request_execution.rs

@@ -0,0 +1,283 @@
+//! Request execution stage: Execute gRPC requests (single or dual dispatch)
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::{error, info_span, Instrument};
+
+use super::PipelineStage;
+use crate::routers::{
+    error,
+    grpc::{
+        context::{ClientSelection, ExecutionResult, LoadGuards, RequestContext, WorkerSelection},
+        proto_wrapper::{
+            ProtoEmbedRequest, ProtoEmbedResponseVariant, ProtoGenerateRequest, ProtoRequest,
+            ProtoStream,
+        },
+    },
+};
+
+type StreamResult = Result<ProtoStream, Box<dyn std::error::Error + Send + Sync>>;
+
+/// Request execution stage: Execute gRPC requests (single or dual dispatch)
+pub(crate) struct RequestExecutionStage {
+    mode: ExecutionMode,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum ExecutionMode {
+    /// Regular mode: single worker execution
+    Single,
+    /// PD mode: dual dispatch to prefill + decode workers
+    DualDispatch,
+}
+
+impl RequestExecutionStage {
+    pub fn new(mode: ExecutionMode) -> Self {
+        Self { mode }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for RequestExecutionStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let proto_request = ctx.state.proto_request.take().ok_or_else(|| {
+            error!(
+                function = "RequestExecutionStage::execute",
+                "Proto request not built"
+            );
+            error::internal_error("proto_request_not_built", "Proto request not built")
+        })?;
+
+        let clients = ctx.state.clients.as_mut().ok_or_else(|| {
+            error!(
+                function = "RequestExecutionStage::execute",
+                "Client acquisition not completed"
+            );
+            error::internal_error(
+                "client_acquisition_not_completed",
+                "Client acquisition not completed",
+            )
+        })?;
+
+        // Create load guards for worker load tracking (increment load when created)
+        // They will be automatically dropped (and decrement load) when RequestContext is dropped
+        let workers = ctx.state.workers.as_ref().ok_or_else(|| {
+            error!(
+                function = "RequestExecutionStage::execute",
+                "Worker selection not completed"
+            );
+            error::internal_error(
+                "worker_selection_not_completed",
+                "Worker selection not completed",
+            )
+        })?;
+
+        ctx.state.load_guards = Some(LoadGuards::new(workers, ctx.input.headers.as_ref()));
+
+        // Extract dispatch metadata for tracing span
+        let request_id = ctx
+            .state
+            .dispatch
+            .as_ref()
+            .map(|d| d.request_id.as_str())
+            .unwrap_or("unknown");
+        let model = ctx
+            .state
+            .dispatch
+            .as_ref()
+            .map(|d| d.model.as_str())
+            .unwrap_or("unknown");
+
+        // Create OTEL span for gRPC request execution
+        let span = info_span!(
+            target: "smg::otel-trace",
+            "grpc_generate",
+            request_id = %request_id,
+            model = %model,
+            mode = ?self.mode,
+        );
+
+        let result = async {
+            match proto_request {
+                ProtoRequest::Generate(req) => match self.mode {
+                    ExecutionMode::Single => self.execute_single(req, clients, workers).await,
+                    ExecutionMode::DualDispatch => {
+                        self.execute_dual_dispatch(req, clients, workers).await
+                    }
+                },
+                ProtoRequest::Embed(req) => self.execute_single_embed(req, clients).await,
+            }
+        }
+        .instrument(span)
+        .await?;
+
+        // Store result in context for ResponseProcessingStage
+        ctx.state.response.execution_result = Some(result);
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "RequestExecution"
+    }
+}
+
+impl RequestExecutionStage {
+    async fn execute_single(
+        &self,
+        proto_request: ProtoGenerateRequest,
+        clients: &mut ClientSelection,
+        workers: &WorkerSelection,
+    ) -> Result<ExecutionResult, Response> {
+        let client = clients.single_mut().ok_or_else(|| {
+            error!(
+                function = "execute_single",
+                "Expected single client but got dual"
+            );
+            error::internal_error(
+                "expected_single_client_got_dual",
+                "Expected single client but got dual",
+            )
+        })?;
+
+        let result = client.generate(proto_request).await;
+
+        // Record circuit breaker outcome
+        workers.record_outcome(result.is_ok());
+
+        let stream = result.map_err(|e| {
+            error!(
+                function = "execute_single",
+                error = %e,
+                "Failed to start generation"
+            );
+            error::internal_error(
+                "start_generation_failed",
+                format!("Failed to start generation: {}", e),
+            )
+        })?;
+
+        Ok(ExecutionResult::Single { stream })
+    }
+
+    async fn execute_single_embed(
+        &self,
+        proto_request: ProtoEmbedRequest,
+        clients: &mut ClientSelection,
+    ) -> Result<ExecutionResult, Response> {
+        let client = clients.single_mut().ok_or_else(|| {
+            error!(
+                function = "execute_single_embed",
+                "Expected single client but got dual"
+            );
+            error::internal_error(
+                "expected_single_client_got_dual",
+                "Expected single client but got dual",
+            )
+        })?;
+
+        let response = client.embed(proto_request).await.map_err(|e| {
+            error!(
+                function = "execute_single_embed",
+                error = %e,
+                "Failed to start embedding"
+            );
+            error::internal_error(
+                "start_embedding_failed",
+                format!("Failed to start embedding: {}", e),
+            )
+        })?;
+
+        match response.into_response() {
+            ProtoEmbedResponseVariant::Complete(complete) => {
+                Ok(ExecutionResult::Embedding { response: complete })
+            }
+            ProtoEmbedResponseVariant::Error(e) => {
+                error!(
+                    function = "execute_single_embed",
+                    error = %e.message(),
+                    "Embedding execution failed"
+                );
+                Err(error::internal_error(
+                    "embedding_execution_failed",
+                    e.message().to_string(),
+                ))
+            }
+            ProtoEmbedResponseVariant::None => {
+                error!(
+                    function = "execute_single_embed",
+                    "Embedding execution returned no response"
+                );
+                Err(error::internal_error(
+                    "embedding_no_response",
+                    "Embedding execution returned no response",
+                ))
+            }
+        }
+    }
+
+    async fn execute_dual_dispatch(
+        &self,
+        proto_request: ProtoGenerateRequest,
+        clients: &mut ClientSelection,
+        workers: &WorkerSelection,
+    ) -> Result<ExecutionResult, Response> {
+        let (prefill_client, decode_client) = clients.dual_mut().ok_or_else(|| {
+            error!(
+                function = "execute_dual_dispatch",
+                "Expected dual clients but got single"
+            );
+            error::internal_error(
+                "expected_dual_clients_got_single",
+                "Expected dual clients but got single",
+            )
+        })?;
+
+        let prefill_request = proto_request.clone_inner();
+        let decode_request = proto_request;
+
+        let (prefill_result, decode_result): (StreamResult, StreamResult) = tokio::join!(
+            prefill_client.generate(prefill_request),
+            decode_client.generate(decode_request)
+        );
+
+        // Record circuit breaker outcomes for each worker individually
+        workers.record_dual_outcomes(prefill_result.is_ok(), decode_result.is_ok());
+
+        // Handle prefill result
+        let prefill_stream = match prefill_result {
+            Ok(s) => s,
+            Err(e) => {
+                error!(
+                    function = "execute_dual_dispatch",
+                    error = %e,
+                    "Prefill worker failed to start"
+                );
+                return Err(error::internal_error(
+                    "prefill_worker_failed_to_start",
+                    format!("Prefill worker failed to start: {}", e),
+                ));
+            }
+        };
+
+        // Handle decode result
+        let decode_stream = match decode_result {
+            Ok(s) => s,
+            Err(e) => {
+                error!(
+                    function = "execute_dual_dispatch",
+                    error = %e,
+                    "Decode worker failed to start"
+                );
+                return Err(error::internal_error(
+                    "decode_worker_failed_to_start",
+                    format!("Decode worker failed to start: {}", e),
+                ));
+            }
+        };
+
+        Ok(ExecutionResult::Dual {
+            prefill: prefill_stream,
+            decode: Box::new(decode_stream),
+        })
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/common/stages/worker_selection.rs

@@ -0,0 +1,275 @@
+//! Worker selection stage: Select appropriate worker(s) based on routing mode
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::{error, warn};
+
+use super::PipelineStage;
+use crate::{
+    core::{ConnectionMode, Worker, WorkerRegistry, WorkerType, UNKNOWN_MODEL_ID},
+    observability::metrics::{metrics_labels, Metrics},
+    policies::{PolicyRegistry, SelectWorkerInfo},
+    routers::{
+        error,
+        grpc::context::{RequestContext, WorkerSelection},
+    },
+};
+
+/// Worker selection stage: Select appropriate worker(s) based on routing mode
+pub(crate) struct WorkerSelectionStage {
+    worker_registry: Arc<WorkerRegistry>,
+    policy_registry: Arc<PolicyRegistry>,
+    mode: WorkerSelectionMode,
+}
+
+pub(crate) enum WorkerSelectionMode {
+    /// Regular mode: select single worker
+    Regular,
+    /// PD mode: select prefill + decode workers
+    PrefillDecode,
+}
+
+impl WorkerSelectionStage {
+    pub fn new(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+        mode: WorkerSelectionMode,
+    ) -> Self {
+        Self {
+            worker_registry,
+            policy_registry,
+            mode,
+        }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for WorkerSelectionStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let prep = ctx.state.preparation.as_ref().ok_or_else(|| {
+            error!(
+                function = "WorkerSelectionStage::execute",
+                "Preparation stage not completed"
+            );
+            error::internal_error(
+                "preparation_stage_not_completed",
+                "Preparation stage not completed",
+            )
+        })?;
+
+        // For Harmony, use selection_text produced during Harmony encoding
+        // Otherwise, use original_text from regular preparation
+        let text = if prep.harmony_mode {
+            prep.selection_text.as_deref()
+        } else {
+            prep.original_text.as_deref()
+        };
+
+        // Get tokens for PrefixHash policy support
+        let tokens = if prep.token_ids.is_empty() {
+            None
+        } else {
+            Some(prep.token_ids.as_slice())
+        };
+
+        let headers = ctx.input.headers.as_ref();
+
+        let workers = match self.mode {
+            WorkerSelectionMode::Regular => {
+                match self
+                    .select_single_worker(ctx.input.model_id.as_deref(), text, tokens, headers)
+                    .await
+                {
+                    Some(w) => WorkerSelection::Single { worker: w },
+                    None => {
+                        let model = ctx.input.model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID);
+                        error!(
+                            function = "WorkerSelectionStage::execute",
+                            mode = "Regular",
+                            model_id = %model,
+                            "No available workers for model"
+                        );
+                        return Err(error::service_unavailable(
+                            "no_available_workers",
+                            format!("No available workers for model: {}", model),
+                        ));
+                    }
+                }
+            }
+            WorkerSelectionMode::PrefillDecode => {
+                match self
+                    .select_pd_pair(ctx.input.model_id.as_deref(), text, tokens, headers)
+                    .await
+                {
+                    Some((prefill, decode)) => WorkerSelection::Dual { prefill, decode },
+                    None => {
+                        let model = ctx.input.model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID);
+                        error!(
+                            function = "WorkerSelectionStage::execute",
+                            mode = "PrefillDecode",
+                            model_id = %model,
+                            "No available PD worker pairs for model"
+                        );
+                        return Err(error::service_unavailable(
+                            "no_available_pd_worker_pairs",
+                            format!("No available PD worker pairs for model: {}", model),
+                        ));
+                    }
+                }
+            }
+        };
+
+        ctx.state.workers = Some(workers);
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "WorkerSelection"
+    }
+}
+
+impl WorkerSelectionStage {
+    async fn select_single_worker(
+        &self,
+        model_id: Option<&str>,
+        text: Option<&str>,
+        tokens: Option<&[u32]>,
+        headers: Option<&http::HeaderMap>,
+    ) -> Option<Arc<dyn Worker>> {
+        // Get workers for the specified model, filtered by connection mode
+        let workers = self.worker_registry.get_workers_filtered(
+            model_id,
+            Some(WorkerType::Regular),
+            Some(ConnectionMode::Grpc { port: None }),
+            None,  // any runtime type
+            false, // get all workers, we'll filter by is_available() next
+        );
+
+        // Use into_iter() to take ownership of Arcs without cloning (avoids atomic inc/dec)
+        let available: Vec<Arc<dyn Worker>> =
+            workers.into_iter().filter(|w| w.is_available()).collect();
+
+        if available.is_empty() {
+            return None;
+        }
+
+        // Get the appropriate policy for this model
+        let policy = match model_id {
+            Some(model) => self.policy_registry.get_policy_or_default(model),
+            None => self.policy_registry.get_default_policy(),
+        };
+
+        // Get cached hash ring for consistent hashing (O(log n) lookup)
+        let hash_ring = self
+            .worker_registry
+            .get_hash_ring(model_id.unwrap_or(UNKNOWN_MODEL_ID));
+
+        // Select worker using the policy
+        let idx = policy
+            .select_worker(
+                &available,
+                &SelectWorkerInfo {
+                    request_text: text,
+                    tokens,
+                    headers,
+                    hash_ring,
+                },
+            )
+            .await?;
+        let selected = available[idx].clone();
+
+        // Record worker selection metric
+        Metrics::record_worker_selection(
+            metrics_labels::WORKER_REGULAR,
+            metrics_labels::CONNECTION_GRPC,
+            model_id.unwrap_or(UNKNOWN_MODEL_ID),
+            policy.name(),
+        );
+
+        Some(selected)
+    }
+
+    async fn select_pd_pair(
+        &self,
+        model_id: Option<&str>,
+        text: Option<&str>,
+        tokens: Option<&[u32]>,
+        headers: Option<&http::HeaderMap>,
+    ) -> Option<(Arc<dyn Worker>, Arc<dyn Worker>)> {
+        let all_workers = self.worker_registry.get_workers_filtered(
+            model_id,
+            None,
+            Some(ConnectionMode::Grpc { port: None }), // Match any gRPC worker
+            None,                                      // any runtime type
+            false,
+        );
+
+        let (available_prefill, available_decode): (Vec<_>, Vec<_>) =
+            all_workers
+                .into_iter()
+                .fold((Vec::new(), Vec::new()), |mut acc, w| {
+                    if w.is_available() {
+                        match w.metadata().worker_type {
+                            WorkerType::Prefill { .. } => acc.0.push(w),
+                            WorkerType::Decode => acc.1.push(w),
+                            _ => {}
+                        }
+                    }
+                    acc
+                });
+
+        if available_prefill.is_empty() {
+            warn!("No available prefill workers");
+            return None;
+        }
+
+        if available_decode.is_empty() {
+            warn!("No available decode workers");
+            return None;
+        }
+
+        // Select using policies
+        let policy = match model_id {
+            Some(model) => self.policy_registry.get_policy_or_default(model),
+            None => self.policy_registry.get_default_policy(),
+        };
+
+        // Get cached hash ring for consistent hashing (O(log n) lookup)
+        let hash_ring = self
+            .worker_registry
+            .get_hash_ring(model_id.unwrap_or(UNKNOWN_MODEL_ID));
+
+        let info = SelectWorkerInfo {
+            request_text: text,
+            tokens,
+            headers,
+            hash_ring,
+        };
+        let prefill_idx = policy.select_worker(&available_prefill, &info).await?;
+        let decode_idx = policy.select_worker(&available_decode, &info).await?;
+
+        let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
+        let policy_name = policy.name();
+
+        // Record worker selection metrics for both prefill and decode
+        Metrics::record_worker_selection(
+            metrics_labels::WORKER_PREFILL,
+            metrics_labels::CONNECTION_GRPC,
+            model,
+            policy_name,
+        );
+        Metrics::record_worker_selection(
+            metrics_labels::WORKER_DECODE,
+            metrics_labels::CONNECTION_GRPC,
+            model,
+            policy_name,
+        );
+
+        Some((
+            available_prefill[prefill_idx].clone(),
+            available_decode[decode_idx].clone(),
+        ))
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/context.rs

@@ -0,0 +1,486 @@
+//! Request context types for gRPC router pipeline
+//!
+//! This module provides the core context types that flow through the router pipeline,
+//! eliminating deep parameter passing chains and providing a single source of truth
+//! for request state.
+
+use std::sync::Arc;
+
+use axum::http::HeaderMap;
+
+use super::{
+    client::GrpcClient,
+    proto_wrapper::{ProtoEmbedComplete, ProtoRequest, ProtoStream},
+};
+use crate::{
+    core::{Worker, WorkerLoadGuard},
+    protocols::{
+        chat::{ChatCompletionRequest, ChatCompletionResponse},
+        classify::{ClassifyRequest, ClassifyResponse},
+        embedding::{EmbeddingRequest, EmbeddingResponse},
+        generate::{GenerateRequest, GenerateResponse},
+        responses::ResponsesRequest,
+    },
+    reasoning_parser::ParserFactory as ReasoningParserFactory,
+    tokenizer::{stop::StopSequenceDecoder, traits::Tokenizer, TokenizerRegistry},
+    tool_parser::ParserFactory as ToolParserFactory,
+};
+
+/// Main request processing context
+///
+/// This is the single source of truth for all request state as it flows
+/// through the pipeline stages. Uses Rust's type system to enforce proper
+/// stage ordering at compile time.
+pub(crate) struct RequestContext {
+    pub input: RequestInput,
+    pub components: Arc<SharedComponents>,
+    pub state: ProcessingState,
+}
+
+/// Immutable request input
+pub(crate) struct RequestInput {
+    pub request_type: RequestType,
+    pub headers: Option<HeaderMap>,
+    pub model_id: Option<String>,
+}
+
+/// Request type variants
+/// Using Arc instead of Box to enable cheap cloning for background tasks
+pub(crate) enum RequestType {
+    Chat(Arc<ChatCompletionRequest>),
+    Generate(Arc<GenerateRequest>),
+    Responses(Arc<ResponsesRequest>),
+    Embedding(Arc<EmbeddingRequest>),
+    Classify(Arc<ClassifyRequest>),
+}
+
+/// Shared components (injected once at creation)
+pub(crate) struct SharedComponents {
+    pub tokenizer_registry: Arc<TokenizerRegistry>,
+    #[allow(dead_code)]
+    pub tool_parser_factory: ToolParserFactory,
+    #[allow(dead_code)]
+    pub reasoning_parser_factory: ReasoningParserFactory,
+}
+
+/// Mutable processing state (evolves through pipeline stages)
+#[derive(Default)]
+pub(crate) struct ProcessingState {
+    // Stage 1: Preparation outputs
+    pub preparation: Option<PreparationOutput>,
+
+    /// Resolved tokenizer (set once in preparation, reused in response processing)
+    /// This avoids redundant registry lookups across pipeline stages.
+    pub tokenizer: Option<Arc<dyn Tokenizer>>,
+
+    // Stage 2: Worker selection outputs
+    pub workers: Option<WorkerSelection>,
+
+    // Stage 3: Client acquisition outputs
+    pub clients: Option<ClientSelection>,
+
+    // Stage 4: Request building outputs
+    pub proto_request: Option<ProtoRequest>,
+
+    // Stage 5: Dispatch metadata
+    pub dispatch: Option<DispatchMetadata>,
+
+    // Load guard for worker load tracking (created at execution stage)
+    pub load_guards: Option<LoadGuards>,
+
+    // Stage 6: Response processing state
+    pub response: ResponseState,
+}
+
+/// Output from preparation stage (Step 1)
+pub(crate) struct PreparationOutput {
+    /// Original text (for chat) or resolved text (for generate)
+    pub original_text: Option<String>,
+
+    /// Tokenized input
+    pub token_ids: Vec<u32>,
+
+    /// Processed messages (chat only)
+    pub processed_messages: Option<super::ProcessedMessages>,
+
+    /// Tool call constraints (if applicable)
+    pub tool_constraints: Option<(String, String)>,
+
+    /// Filtered request (if tools were filtered)
+    pub filtered_request: Option<ChatCompletionRequest>,
+
+    // Harmony-specific fields
+    /// Whether this is a Harmony request (default: false)
+    pub harmony_mode: bool,
+
+    /// Selection text for worker routing (Harmony only)
+    pub selection_text: Option<String>,
+
+    /// Harmony messages for history tracking (Harmony only)
+    #[allow(dead_code)]
+    pub harmony_messages: Option<Vec<super::harmony::HarmonyMessage>>,
+
+    /// Stop token IDs for Harmony models
+    pub harmony_stop_ids: Option<Vec<u32>>,
+}
+
+/// Worker selection (Step 2)
+pub(crate) enum WorkerSelection {
+    Single {
+        worker: Arc<dyn Worker>,
+    },
+    Dual {
+        prefill: Arc<dyn Worker>,
+        decode: Arc<dyn Worker>,
+    },
+}
+
+/// Client selection (Step 3)
+pub(crate) enum ClientSelection {
+    Single {
+        client: GrpcClient,
+    },
+    Dual {
+        prefill: GrpcClient,
+        decode: GrpcClient,
+    },
+}
+
+/// Dispatch metadata (Step 5)
+#[derive(Clone)]
+pub(crate) struct DispatchMetadata {
+    pub request_id: String,
+    pub model: String,
+    pub created: u64,
+    pub weight_version: Option<String>,
+}
+
+/// Load guards for worker load tracking
+/// Automatically decrements load when dropped
+pub(crate) enum LoadGuards {
+    Single {
+        _guard: WorkerLoadGuard,
+    },
+    Dual {
+        _prefill: WorkerLoadGuard,
+        _decode: WorkerLoadGuard,
+    },
+}
+
+impl LoadGuards {
+    pub fn new(selection: &WorkerSelection, headers: Option<&HeaderMap>) -> Self {
+        match selection {
+            WorkerSelection::Single { worker } => LoadGuards::Single {
+                _guard: WorkerLoadGuard::new(worker.clone(), headers),
+            },
+            WorkerSelection::Dual { prefill, decode } => LoadGuards::Dual {
+                _prefill: WorkerLoadGuard::new(prefill.clone(), headers),
+                _decode: WorkerLoadGuard::new(decode.clone(), headers),
+            },
+        }
+    }
+}
+
+/// Response processing state (Step 6)
+#[derive(Default)]
+pub(crate) struct ResponseState {
+    /// Stop sequence decoder
+    pub stop_decoder: Option<StopSequenceDecoder>,
+
+    /// Execution result (streams from workers)
+    pub execution_result: Option<ExecutionResult>,
+
+    /// Final processed response
+    pub final_response: Option<FinalResponse>,
+
+    /// Responses API iteration result (Harmony only, for tool loop orchestration)
+    pub responses_iteration_result: Option<super::harmony::ResponsesIterationResult>,
+}
+
+impl RequestContext {
+    /// Create context for chat completion request
+    pub fn for_chat(
+        request: Arc<ChatCompletionRequest>,
+        headers: Option<HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Self {
+        Self {
+            input: RequestInput {
+                request_type: RequestType::Chat(request),
+                headers,
+                model_id,
+            },
+            components,
+            state: ProcessingState::default(),
+        }
+    }
+
+    /// Create context for generate request
+    pub fn for_generate(
+        request: Arc<GenerateRequest>,
+        headers: Option<HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Self {
+        Self {
+            input: RequestInput {
+                request_type: RequestType::Generate(request),
+                headers,
+                model_id,
+            },
+            components,
+            state: ProcessingState::default(),
+        }
+    }
+
+    /// Create context for Responses API request
+    pub fn for_responses(
+        request: Arc<ResponsesRequest>,
+        headers: Option<HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Self {
+        Self {
+            input: RequestInput {
+                request_type: RequestType::Responses(request),
+                headers,
+                model_id,
+            },
+            components,
+            state: ProcessingState::default(),
+        }
+    }
+
+    /// Create context for embedding request
+    pub fn for_embedding(
+        request: Arc<EmbeddingRequest>,
+        headers: Option<HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Self {
+        Self {
+            input: RequestInput {
+                request_type: RequestType::Embedding(request),
+                headers,
+                model_id,
+            },
+            components,
+            state: ProcessingState::default(),
+        }
+    }
+
+    /// Create context for classify request
+    pub fn for_classify(
+        request: Arc<ClassifyRequest>,
+        headers: Option<HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Self {
+        Self {
+            input: RequestInput {
+                request_type: RequestType::Classify(request),
+                headers,
+                model_id,
+            },
+            components,
+            state: ProcessingState::default(),
+        }
+    }
+
+    /// Get chat request (panics if not chat)
+    pub fn chat_request(&self) -> &ChatCompletionRequest {
+        match &self.input.request_type {
+            RequestType::Chat(req) => req.as_ref(),
+            _ => panic!("Expected chat request"),
+        }
+    }
+
+    /// Get Arc clone of chat request (panics if not chat)
+    pub fn chat_request_arc(&self) -> Arc<ChatCompletionRequest> {
+        match &self.input.request_type {
+            RequestType::Chat(req) => Arc::clone(req),
+            _ => panic!("Expected chat request"),
+        }
+    }
+
+    /// Get generate request (panics if not generate)
+    pub fn generate_request(&self) -> &GenerateRequest {
+        match &self.input.request_type {
+            RequestType::Generate(req) => req.as_ref(),
+            _ => panic!("Expected generate request"),
+        }
+    }
+
+    /// Get Arc clone of generate request (panics if not generate)
+    pub fn generate_request_arc(&self) -> Arc<GenerateRequest> {
+        match &self.input.request_type {
+            RequestType::Generate(req) => Arc::clone(req),
+            _ => panic!("Expected generate request"),
+        }
+    }
+
+    /// Get Arc clone of responses request (panics if not responses)
+    pub fn responses_request_arc(&self) -> Arc<ResponsesRequest> {
+        match &self.input.request_type {
+            RequestType::Responses(req) => Arc::clone(req),
+            _ => panic!("Expected responses request"),
+        }
+    }
+
+    /// Check if request is streaming
+    pub fn is_streaming(&self) -> bool {
+        match &self.input.request_type {
+            RequestType::Chat(req) => req.stream,
+            RequestType::Generate(req) => req.stream,
+            RequestType::Responses(req) => req.stream.unwrap_or(false),
+            RequestType::Embedding(_) => false, // Embeddings are never streaming
+            RequestType::Classify(_) => false,  // Classification is never streaming
+        }
+    }
+
+    /// Get the cached tokenizer, cloning the Arc (cheap 8-byte clone)
+    ///
+    /// Returns None if tokenizer hasn't been resolved yet.
+    /// The tokenizer is resolved once in the preparation stage and cached for reuse.
+    pub fn tokenizer_arc(&self) -> Option<Arc<dyn Tokenizer>> {
+        self.state.tokenizer.clone()
+    }
+}
+
+/// Some methods are kept for API completeness even if currently unused.
+#[allow(dead_code)]
+impl WorkerSelection {
+    pub fn is_dual(&self) -> bool {
+        matches!(self, Self::Dual { .. })
+    }
+
+    pub fn single(&self) -> Option<&Arc<dyn Worker>> {
+        match self {
+            Self::Single { worker } => Some(worker),
+            _ => None,
+        }
+    }
+
+    /// Record circuit breaker outcome for all workers
+    pub fn record_outcome(&self, success: bool) {
+        match self {
+            Self::Single { worker } => worker.record_outcome(success),
+            Self::Dual { prefill, decode } => {
+                prefill.record_outcome(success);
+                decode.record_outcome(success);
+            }
+        }
+    }
+
+    /// Record circuit breaker outcomes for dual dispatch (individual tracking)
+    pub fn record_dual_outcomes(&self, prefill_success: bool, decode_success: bool) {
+        if let Self::Dual { prefill, decode } = self {
+            prefill.record_outcome(prefill_success);
+            decode.record_outcome(decode_success);
+        }
+    }
+
+    #[allow(clippy::type_complexity)]
+    pub fn dual(&self) -> Option<(&Arc<dyn Worker>, &Arc<dyn Worker>)> {
+        match self {
+            Self::Dual { prefill, decode } => Some((prefill, decode)),
+            _ => None,
+        }
+    }
+
+    pub fn prefill_worker(&self) -> Option<&Arc<dyn Worker>> {
+        match self {
+            Self::Dual { prefill, .. } => Some(prefill),
+            _ => None,
+        }
+    }
+
+    pub fn decode_worker(&self) -> Option<&Arc<dyn Worker>> {
+        match self {
+            Self::Dual { decode, .. } => Some(decode),
+            _ => None,
+        }
+    }
+}
+
+/// Some methods are kept for API completeness even if currently unused.
+#[allow(dead_code)]
+impl ClientSelection {
+    pub fn single(&self) -> Option<&GrpcClient> {
+        match self {
+            Self::Single { client } => Some(client),
+            _ => None,
+        }
+    }
+
+    pub fn single_mut(&mut self) -> Option<&mut GrpcClient> {
+        match self {
+            Self::Single { client } => Some(client),
+            _ => None,
+        }
+    }
+
+    pub fn dual_mut(&mut self) -> Option<(&mut GrpcClient, &mut GrpcClient)> {
+        match self {
+            Self::Dual { prefill, decode } => Some((prefill, decode)),
+            _ => None,
+        }
+    }
+
+    pub fn prefill_client(&self) -> Option<&GrpcClient> {
+        match self {
+            Self::Dual { prefill, .. } => Some(prefill),
+            _ => None,
+        }
+    }
+
+    pub fn prefill_client_mut(&mut self) -> Option<&mut GrpcClient> {
+        match self {
+            Self::Dual { prefill, .. } => Some(prefill),
+            _ => None,
+        }
+    }
+
+    pub fn decode_client(&self) -> Option<&GrpcClient> {
+        match self {
+            Self::Dual { decode, .. } => Some(decode),
+            _ => None,
+        }
+    }
+
+    pub fn decode_client_mut(&mut self) -> Option<&mut GrpcClient> {
+        match self {
+            Self::Dual { decode, .. } => Some(decode),
+            _ => None,
+        }
+    }
+}
+
+/// Result of request execution (streams from workers)
+/// Uses ProtoStream to automatically abort on cancellation
+pub(crate) enum ExecutionResult {
+    Single {
+        stream: ProtoStream,
+    },
+    Dual {
+        prefill: ProtoStream,
+        decode: Box<ProtoStream>,
+    },
+    /// Embedding requests return a single response, not a stream
+    Embedding {
+        response: ProtoEmbedComplete,
+    },
+}
+
+/// Final processed response
+#[derive(Debug)]
+pub(crate) enum FinalResponse {
+    Chat(ChatCompletionResponse),
+    /// Generate response is a Vec of GenerateResponse (n=1 returns single item, n>1 returns multiple)
+    Generate(Vec<GenerateResponse>),
+    /// Embedding response
+    Embedding(EmbeddingResponse),
+    /// Classification response
+    Classify(ClassifyResponse),
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/builder.rs

@@ -0,0 +1,908 @@
+//! Harmony request builder
+//!
+//! Handles encoding of Chat/Responses requests into Harmony format using openai-harmony library.
+
+use std::sync::OnceLock;
+
+use chrono::Local;
+use openai_harmony::{
+    chat::{
+        Author, ChannelConfig, Content, Conversation, DeveloperContent, Message as HarmonyMessage,
+        ReasoningEffort, Role, SystemContent, TextContent, ToolDescription,
+    },
+    HarmonyEncoding, HarmonyEncodingName,
+};
+use tracing::{debug, trace};
+
+use super::types::HarmonyBuildOutput;
+use crate::protocols::{
+    chat::{ChatCompletionRequest, ChatMessage, MessageContent},
+    common::{ContentPart, Tool},
+    responses::{
+        ReasoningEffort as ResponsesReasoningEffort, ResponseContentPart, ResponseInput,
+        ResponseInputOutputItem, ResponseReasoningContent, ResponseTool, ResponseToolType,
+        ResponsesRequest, StringOrContentParts,
+    },
+};
+
+/// Global Harmony encoding (lazy-initialized)
+static HARMONY_ENCODING: OnceLock<HarmonyEncoding> = OnceLock::new();
+
+/// Get or initialize the Harmony encoding
+///
+/// Uses HarmonyGptOss encoding which supports the gpt-oss model family.
+pub(super) fn get_harmony_encoding() -> &'static HarmonyEncoding {
+    HARMONY_ENCODING.get_or_init(|| {
+        tokio::task::block_in_place(|| {
+            openai_harmony::load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss)
+                .expect("Failed to load Harmony encoding")
+        })
+    })
+}
+
+/// Built-in tools that are added to the system message
+const BUILTIN_TOOLS: &[&str] = &["web_search_preview", "code_interpreter", "container"];
+
+/// Trait for tool-like objects that can be converted to Harmony ToolDescription
+trait ToolLike {
+    /// Check if this is a built-in tool (should be skipped in developer message)
+    #[allow(dead_code)]
+    fn is_builtin(&self) -> bool;
+
+    /// Check if this is a custom tool (function or MCP)
+    fn is_custom(&self) -> bool;
+
+    /// Convert to ToolDescription
+    fn to_tool_description(&self) -> Option<ToolDescription>;
+}
+
+/// Implement ToolLike for Chat Completion Tool
+impl ToolLike for Tool {
+    fn is_builtin(&self) -> bool {
+        matches!(
+            self.tool_type.as_str(),
+            "web_search_preview" | "code_interpreter" | "container"
+        )
+    }
+
+    fn is_custom(&self) -> bool {
+        matches!(self.tool_type.as_str(), "mcp" | "function")
+    }
+
+    fn to_tool_description(&self) -> Option<ToolDescription> {
+        Some(ToolDescription::new(
+            self.function.name.clone(),
+            self.function.description.clone().unwrap_or_default(),
+            Some(self.function.parameters.clone()),
+        ))
+    }
+}
+
+/// Implement ToolLike for Responses API Tool
+impl ToolLike for ResponseTool {
+    fn is_builtin(&self) -> bool {
+        matches!(
+            self.r#type,
+            ResponseToolType::WebSearchPreview | ResponseToolType::CodeInterpreter
+        )
+    }
+
+    fn is_custom(&self) -> bool {
+        matches!(
+            self.r#type,
+            ResponseToolType::Mcp | ResponseToolType::Function
+        )
+    }
+
+    fn to_tool_description(&self) -> Option<ToolDescription> {
+        self.function.as_ref().map(|func| {
+            ToolDescription::new(
+                func.name.clone(),
+                func.description.clone().unwrap_or_default(),
+                Some(func.parameters.clone()),
+            )
+        })
+    }
+}
+
+fn has_custom_tools(tool_types: &[&str]) -> bool {
+    !tool_types.iter().all(|t| BUILTIN_TOOLS.contains(t))
+}
+
+/// Harmony request builder
+///
+/// Converts OpenAI-format requests into Harmony-encoded format with input_ids,
+/// stop tokens, and selection text for worker routing.
+pub(crate) struct HarmonyBuilder {
+    encoding: &'static HarmonyEncoding,
+}
+
+impl HarmonyBuilder {
+    /// Create a new Harmony builder
+    pub fn new() -> Self {
+        Self {
+            encoding: get_harmony_encoding(),
+        }
+    }
+
+    /// Build Harmony request from Chat Completion request
+    ///
+    /// # Arguments
+    ///
+    /// * `request` - The ChatCompletionRequest to encode
+    ///
+    /// # Returns
+    ///
+    /// HarmonyBuildOutput containing input_ids, stop_token_ids, selection_text, and messages
+    pub fn build_from_chat(
+        &self,
+        request: &ChatCompletionRequest,
+    ) -> Result<HarmonyBuildOutput, String> {
+        let mut all_messages = Vec::new();
+
+        let sys_msg = self.build_system_message_from_chat(request);
+        all_messages.push(sys_msg);
+
+        let dev_msg = self.build_developer_message_from_chat(request.tools.as_ref());
+        all_messages.push(dev_msg);
+
+        let mut user_messages = self.convert_chat_messages(&request.messages)?;
+        all_messages.append(&mut user_messages);
+
+        let conversation = Conversation::from_messages(all_messages.clone());
+        let token_ids = self
+            .encoding
+            .render_conversation_for_completion(&conversation, Role::Assistant, None)
+            .map_err(|e| format!("Failed to encode Harmony conversation: {}", e))?;
+
+        let selection_text = self.extract_selection_text(&all_messages);
+
+        // Get stop tokens for Harmony assistant actions (<|return|> and <|call|>)
+        let stop_token_ids: Vec<u32> = self
+            .encoding
+            .stop_tokens_for_assistant_actions()
+            .into_iter()
+            .flat_map(|set| set.into_iter())
+            .collect();
+
+        Ok(HarmonyBuildOutput {
+            input_ids: token_ids,
+            stop_token_ids,
+            selection_text,
+            harmony_messages: all_messages
+                .into_iter()
+                .map(super::types::HarmonyMessage::from_openai_harmony)
+                .collect(),
+        })
+    }
+
+    /// Build Harmony request from Responses request
+    ///
+    /// # Arguments
+    ///
+    /// * `request` - The ResponsesRequest to encode
+    ///
+    /// # Returns
+    ///
+    /// HarmonyBuildOutput containing input_ids, stop_token_ids, selection_text, and messages
+    pub fn build_from_responses(
+        &self,
+        request: &ResponsesRequest,
+    ) -> Result<HarmonyBuildOutput, String> {
+        let all_messages = self.construct_input_messages_with_harmony(request)?;
+
+        let conversation = Conversation::from_messages(all_messages.clone());
+        let token_ids = self
+            .encoding
+            .render_conversation_for_completion(&conversation, Role::Assistant, None)
+            .map_err(|e| format!("Failed to encode Harmony conversation: {}", e))?;
+
+        let selection_text = self.extract_selection_text(&all_messages);
+
+        // Get stop tokens for Harmony assistant actions (<|return|> and <|call|>)
+        let stop_token_ids: Vec<u32> = self
+            .encoding
+            .stop_tokens_for_assistant_actions()
+            .into_iter()
+            .flat_map(|set| set.into_iter())
+            .collect();
+
+        // Decode tokens to see what the model actually receives
+        let decoded_text = self
+            .encoding
+            .tokenizer()
+            .decode_utf8(&token_ids)
+            .unwrap_or_else(|_| "<decode error>".to_string());
+        trace!(
+            token_count = token_ids.len(),
+            token_preview = ?&token_ids[..token_ids.len().min(20)],
+            decoded_length = decoded_text.len(),
+            "Encoded conversation to tokens - decoded text follows:"
+        );
+        trace!("DECODED_TEXT_START\n{}\nDECODED_TEXT_END", decoded_text);
+
+        Ok(HarmonyBuildOutput {
+            input_ids: token_ids,
+            stop_token_ids,
+            selection_text,
+            harmony_messages: all_messages
+                .into_iter()
+                .map(super::types::HarmonyMessage::from_openai_harmony)
+                .collect(),
+        })
+    }
+
+    /// Build system message with common logic
+    ///
+    /// # Arguments
+    /// * `reasoning_effort` - Optional reasoning effort level
+    /// * `has_tools` - Whether custom tools are present
+    fn build_system_message(
+        &self,
+        reasoning_effort: Option<ReasoningEffort>,
+        has_tools: bool,
+    ) -> HarmonyMessage {
+        let mut sys_content = SystemContent::new();
+
+        // Add reasoning_effort if provided
+        if let Some(effort) = reasoning_effort {
+            sys_content = sys_content.with_reasoning_effort(effort);
+        }
+
+        // Set conversation start date (always current date)
+        sys_content =
+            sys_content.with_conversation_start_date(Local::now().format("%Y-%m-%d").to_string());
+
+        // If no tools, remove "commentary" from valid channels
+        if !has_tools {
+            if let Some(channel_config) = &sys_content.channel_config {
+                let valid_channels: Vec<String> = channel_config
+                    .valid_channels
+                    .iter()
+                    .filter(|c| c.as_str() != "commentary")
+                    .cloned()
+                    .collect();
+                sys_content = sys_content
+                    .with_channel_config(ChannelConfig::require_channels(valid_channels));
+            }
+        }
+
+        HarmonyMessage::from_role_and_content(Role::System, sys_content)
+    }
+
+    fn build_system_message_from_chat(&self, request: &ChatCompletionRequest) -> HarmonyMessage {
+        let reasoning_effort = request
+            .reasoning_effort
+            .as_deref()
+            .map(|effort| match effort {
+                "high" => ReasoningEffort::High,
+                "medium" => ReasoningEffort::Medium,
+                "low" => ReasoningEffort::Low,
+                // Harmony does not support minimal reasoning effort
+                "minimal" => ReasoningEffort::Low,
+                _ => ReasoningEffort::Medium,
+            });
+
+        let has_tools = request.tools.is_some();
+        self.build_system_message(reasoning_effort, has_tools)
+    }
+
+    /// Build system message from ResponsesRequest
+    ///
+    /// # Arguments
+    /// * `request` - The ResponsesRequest
+    /// * `with_custom_tools` - Whether custom tools (beyond built-ins) are present
+    fn build_system_message_from_responses(
+        &self,
+        request: &ResponsesRequest,
+        with_custom_tools: bool,
+    ) -> HarmonyMessage {
+        let reasoning_effort = request
+            .reasoning
+            .as_ref()
+            .and_then(|r| r.effort.as_ref())
+            .map(|effort| match effort {
+                ResponsesReasoningEffort::High => ReasoningEffort::High,
+                ResponsesReasoningEffort::Medium => ReasoningEffort::Medium,
+                ResponsesReasoningEffort::Low => ReasoningEffort::Low,
+                ResponsesReasoningEffort::Minimal => ReasoningEffort::Low,
+            });
+
+        self.build_system_message(reasoning_effort, with_custom_tools)
+    }
+
+    /// Build developer message with common logic
+    ///
+    /// Filters out built-in tools and converts custom tools to ToolDescription
+    ///
+    /// # Arguments
+    /// * `tools` - Optional list of tools
+    /// * `instructions` - Optional instructions (Responses API only)
+    fn build_developer_message<T: ToolLike>(
+        &self,
+        tools: Option<&Vec<T>>,
+        instructions: Option<&str>,
+    ) -> HarmonyMessage {
+        let mut dev_content = DeveloperContent::new();
+
+        // Add instructions if provided (Responses API only)
+        if let Some(instructions) = instructions {
+            dev_content = dev_content.with_instructions(instructions.to_string());
+        }
+
+        // Early return if no tools
+        let Some(tools) = tools else {
+            return HarmonyMessage::from_role_and_content(Role::Developer, dev_content);
+        };
+
+        // Filter to custom tools and convert to ToolDescription
+        let tool_descriptions: Vec<ToolDescription> = tools
+            .iter()
+            .filter(|t| t.is_custom())
+            .filter_map(|t| t.to_tool_description())
+            .collect();
+
+        // Add function tools to developer content
+        if !tool_descriptions.is_empty() {
+            dev_content = dev_content.with_function_tools(tool_descriptions);
+        }
+
+        HarmonyMessage::from_role_and_content(Role::Developer, dev_content)
+    }
+
+    fn build_developer_message_from_chat(&self, tools: Option<&Vec<Tool>>) -> HarmonyMessage {
+        self.build_developer_message(tools, None)
+    }
+
+    /// Build developer message from Responses request
+    ///
+    /// # Arguments
+    /// * `instructions` - Optional instructions (Responses API specific)
+    /// * `tools` - Optional list of tools
+    fn build_developer_message_from_responses(
+        &self,
+        instructions: Option<&str>,
+        tools: Option<&Vec<ResponseTool>>,
+    ) -> HarmonyMessage {
+        self.build_developer_message(tools, instructions)
+    }
+
+    /// Construct input messages for Responses API with Harmony
+    ///
+    /// Handles both new conversations and continuations of previous responses.
+    ///
+    /// This handles:
+    /// - New conversation: system message, developer message, and user input
+    /// - Continuing conversation: loads previous messages, cleans up chain-of-thoughts
+    /// - MCP tool allowlisting for special tool types
+    /// - Complex response input parsing with function call tracking
+    ///
+    /// # Arguments
+    /// * `request` - The ResponsesRequest
+    /// * `prev_response` - Optional previous response to continue from
+    fn construct_input_messages_with_harmony(
+        &self,
+        request: &ResponsesRequest,
+    ) -> Result<Vec<HarmonyMessage>, String> {
+        let mut all_messages = Vec::new();
+
+        // Handle new vs continuing conversation
+        if request.previous_response_id.is_none() {
+            // New conversation
+
+            let tool_types: Vec<&str> = request
+                .tools
+                .as_ref()
+                .map(|tools| {
+                    tools
+                        .iter()
+                        .map(|tool| match tool.r#type {
+                            ResponseToolType::Function => "function",
+                            ResponseToolType::WebSearchPreview => "web_search_preview",
+                            ResponseToolType::CodeInterpreter => "code_interpreter",
+                            ResponseToolType::Mcp => "mcp",
+                        })
+                        .collect()
+                })
+                .unwrap_or_default();
+
+            let with_custom_tools = has_custom_tools(&tool_types);
+
+            // Add system message
+            let sys_msg = self.build_system_message_from_responses(request, with_custom_tools);
+            all_messages.push(sys_msg);
+
+            // Add developer message only if we have custom tools
+            if with_custom_tools {
+                let dev_msg = self.build_developer_message_from_responses(
+                    request.instructions.as_deref(),
+                    request.tools.as_ref(),
+                );
+                all_messages.push(dev_msg);
+            }
+        } else {
+            // Continue the previous conversation
+            // NOTE: Previous messages are loaded by serve_harmony_responses() before calling this method.
+            // The request.input will already contain the conversation history when previous_response_id was set.
+            // We just proceed with parsing the input items as normal.
+            debug!("Continuing conversation (history already loaded in request.input)");
+        }
+
+        // Append the new input
+        // Responses API supports simple text inputs without chat format
+        match &request.input {
+            ResponseInput::Text(text) => {
+                let user_msg = HarmonyMessage {
+                    author: Author {
+                        role: Role::User,
+                        name: None,
+                    },
+                    recipient: None,
+                    content: vec![Content::Text(TextContent { text: text.clone() })],
+                    channel: None,
+                    content_type: None,
+                };
+                all_messages.push(user_msg);
+            }
+            ResponseInput::Items(items) => {
+                // Track function calls for looking up call_id → name mapping
+                let mut prev_outputs: Vec<&ResponseInputOutputItem> = Vec::new();
+
+                for item in items {
+                    let msg = self.parse_response_item_to_harmony_message(item, &prev_outputs)?;
+                    all_messages.push(msg);
+
+                    // Track function tool calls so that function_call_output can find the name
+                    if matches!(item, ResponseInputOutputItem::FunctionToolCall { .. }) {
+                        prev_outputs.push(item);
+                    }
+                }
+            }
+        }
+
+        debug!(
+            message_count = all_messages.len(),
+            "Constructed Harmony messages for Responses API"
+        );
+        Ok(all_messages)
+    }
+
+    /// Parse a ResponseInputOutputItem into a HarmonyMessage
+    ///
+    /// Handles conversion of various response item types (messages, function calls, reasoning, etc.)
+    /// to Harmony message format.
+    ///
+    /// # Arguments
+    /// * `item` - The ResponseInputOutputItem to parse
+    /// * `prev_outputs` - Previous items for looking up function call names (for function_call_output)
+    fn parse_response_item_to_harmony_message(
+        &self,
+        item: &ResponseInputOutputItem,
+        prev_outputs: &[&ResponseInputOutputItem],
+    ) -> Result<HarmonyMessage, String> {
+        match item {
+            // Regular message (user or assistant)
+            ResponseInputOutputItem::Message { role, content, .. } => {
+                let harmony_role = match role.as_str() {
+                    "user" => Role::User,
+                    "assistant" => Role::Assistant,
+                    "system" => Role::System,
+                    _ => Role::User, // Default to user for unknown roles
+                };
+
+                // Extract text from content parts
+                let text_parts: Vec<String> = content
+                    .iter()
+                    .filter_map(|part| match part {
+                        ResponseContentPart::OutputText { text, .. } => Some(text.clone()),
+                        ResponseContentPart::InputText { text } => Some(text.clone()),
+                        ResponseContentPart::Unknown => None,
+                    })
+                    .collect();
+
+                let text = text_parts.join("\n");
+
+                Ok(HarmonyMessage {
+                    author: Author {
+                        role: harmony_role,
+                        name: None,
+                    },
+                    recipient: None,
+                    content: vec![Content::Text(TextContent { text })],
+                    channel: None,
+                    content_type: None,
+                })
+            }
+
+            // Reasoning content (chain-of-thought)
+            ResponseInputOutputItem::Reasoning { content, .. } => {
+                // Extract reasoning text
+                let reasoning_texts: Vec<String> = content
+                    .iter()
+                    .map(|rc| match rc {
+                        ResponseReasoningContent::ReasoningText { text } => text.clone(),
+                    })
+                    .collect();
+
+                let text = reasoning_texts.join("\n");
+
+                // Reasoning goes in the "analysis" channel for Harmony
+                Ok(HarmonyMessage {
+                    author: Author {
+                        role: Role::Assistant,
+                        name: None,
+                    },
+                    recipient: None,
+                    content: vec![Content::Text(TextContent { text })],
+                    channel: Some("analysis".to_string()),
+                    content_type: None,
+                })
+            }
+
+            // Function tool call (with optional output)
+            ResponseInputOutputItem::FunctionToolCall {
+                name,
+                arguments,
+                output,
+                ..
+            } => {
+                // If there's an output, this represents the tool result
+                // Otherwise, it's the tool call itself
+                if let Some(output_str) = output {
+                    // Tool result - use Tool role with "functions.{name}" as author name
+                    // IMPORTANT: Must include recipient="assistant" for parser to recognize it.
+                    // We keep channel=None to minimize what the model might copy.
+                    let author_name = format!("functions.{}", name);
+                    debug!(
+                        tool_name = %name,
+                        author_name = %author_name,
+                        output_preview = %output_str.chars().take(100).collect::<String>(),
+                        "Building tool result message with Tool role (recipient=assistant, no channel)"
+                    );
+                    Ok(HarmonyMessage {
+                        author: Author {
+                            role: Role::Tool,
+                            name: Some(author_name),
+                        },
+                        recipient: Some("assistant".to_string()),
+                        content: vec![Content::Text(TextContent {
+                            text: output_str.clone(),
+                        })],
+                        channel: None,
+                        content_type: None,
+                    })
+                } else {
+                    // Tool call - assistant message in commentary channel with recipient
+                    // msg.with_channel("commentary").with_recipient(f"functions.{name}")
+                    let recipient = format!("functions.{}", name);
+                    debug!(
+                        tool_name = %name,
+                        recipient = %recipient,
+                        "Building tool call message with recipient"
+                    );
+                    Ok(HarmonyMessage {
+                        author: Author {
+                            role: Role::Assistant,
+                            name: None,
+                        },
+                        recipient: Some(recipient),
+                        content: vec![Content::Text(TextContent {
+                            text: arguments.clone(),
+                        })],
+                        channel: Some("commentary".to_string()),
+                        content_type: Some("json".to_string()),
+                    })
+                }
+            }
+
+            // Function call output (separate from call) - requires looking up the original call
+            ResponseInputOutputItem::FunctionCallOutput {
+                call_id, output, ..
+            } => {
+                // Search prev_outputs in reverse order to find the matching function call
+                let call = prev_outputs
+                    .iter()
+                    .rev()
+                    .find_map(|item| match item {
+                        ResponseInputOutputItem::FunctionToolCall {
+                            call_id: item_call_id,
+                            name,
+                            ..
+                        } if item_call_id == call_id => Some(name.clone()),
+                        _ => None,
+                    })
+                    .ok_or_else(|| format!("No function call found for call_id: {}", call_id))?;
+
+                // Create Tool message with "functions.{name}" prefix
+                // IMPORTANT: Must include recipient="assistant" for parser to recognize it.
+                // We keep channel=None to minimize what the model might copy.
+                Ok(HarmonyMessage {
+                    author: Author {
+                        role: Role::Tool,
+                        name: Some(format!("functions.{}", call)),
+                    },
+                    recipient: Some("assistant".to_string()),
+                    content: vec![Content::Text(TextContent {
+                        text: output.clone(),
+                    })],
+                    channel: None,
+                    content_type: None,
+                })
+            }
+
+            // Simple input message (usually user message)
+            ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => {
+                let harmony_role = match role.as_str() {
+                    "user" => Role::User,
+                    "assistant" => Role::Assistant,
+                    "system" => Role::System,
+                    _ => Role::User,
+                };
+
+                let text = match content {
+                    StringOrContentParts::String(s) => s.clone(),
+                    StringOrContentParts::Array(parts) => {
+                        // Extract text from content parts
+                        parts
+                            .iter()
+                            .filter_map(|part| match part {
+                                ResponseContentPart::OutputText { text, .. } => Some(text.clone()),
+                                ResponseContentPart::InputText { text } => Some(text.clone()),
+                                ResponseContentPart::Unknown => None,
+                            })
+                            .collect::<Vec<_>>()
+                            .join("\n")
+                    }
+                };
+
+                Ok(HarmonyMessage {
+                    author: Author {
+                        role: harmony_role,
+                        name: None,
+                    },
+                    recipient: None,
+                    content: vec![Content::Text(TextContent { text })],
+                    channel: None,
+                    content_type: None,
+                })
+            }
+        }
+    }
+
+    /// Convert OpenAI ChatMessage format to Harmony messages
+    ///
+    /// - Assistant messages with tool_calls create multiple messages (one per tool call)
+    /// - Tool role messages use Role::Tool with proper author
+    /// - Tool-related messages use channel="commentary"
+    fn convert_chat_messages(
+        &self,
+        messages: &[ChatMessage],
+    ) -> Result<Vec<HarmonyMessage>, String> {
+        let mut harmony_messages = Vec::new();
+
+        // Build a map of tool_call_id -> function_name for tool responses
+        let mut tool_call_map = std::collections::HashMap::new();
+        for msg in messages {
+            if let ChatMessage::Assistant {
+                tool_calls: Some(calls),
+                ..
+            } = msg
+            {
+                for call in calls {
+                    tool_call_map.insert(call.id.clone(), call.function.name.clone());
+                }
+            }
+        }
+
+        for msg in messages {
+            match msg {
+                ChatMessage::System { content, name } => {
+                    // System messages stay as-is
+                    let harmony_msg = HarmonyMessage {
+                        author: Author {
+                            role: Role::System,
+                            name: name.clone(),
+                        },
+                        recipient: None,
+                        content: vec![Content::Text(TextContent {
+                            text: content.to_simple_string(),
+                        })],
+                        channel: None,
+                        content_type: None,
+                    };
+                    harmony_messages.push(harmony_msg);
+                }
+                ChatMessage::Developer {
+                    content,
+                    name,
+                    tools: _,
+                } => {
+                    // Developer messages stay as-is
+                    let harmony_msg = HarmonyMessage {
+                        author: Author {
+                            role: Role::Developer,
+                            name: name.clone(),
+                        },
+                        recipient: None,
+                        content: vec![Content::Text(TextContent {
+                            text: content.to_simple_string(),
+                        })],
+                        channel: None,
+                        content_type: None,
+                    };
+                    harmony_messages.push(harmony_msg);
+                }
+
+                ChatMessage::User { content, name } => {
+                    // Extract text from user content
+                    let text = match content {
+                        MessageContent::Text(text) => text.clone(),
+                        MessageContent::Parts(parts) => {
+                            // For multimodal content, extract text parts
+                            parts
+                                .iter()
+                                .filter_map(|part| {
+                                    if let ContentPart::Text { text } = part {
+                                        Some(text.as_str())
+                                    } else {
+                                        None
+                                    }
+                                })
+                                .collect::<Vec<_>>()
+                                .join("\n")
+                        }
+                    };
+
+                    let harmony_msg = HarmonyMessage {
+                        author: Author {
+                            role: Role::User,
+                            name: name.clone(),
+                        },
+                        recipient: None,
+                        content: vec![Content::Text(TextContent { text })],
+                        channel: None,
+                        content_type: None,
+                    };
+                    harmony_messages.push(harmony_msg);
+                }
+
+                ChatMessage::Assistant {
+                    content,
+                    name,
+                    tool_calls,
+                    reasoning_content,
+                } => {
+                    if let Some(calls) = tool_calls {
+                        // Create one message per tool call with channel="commentary"
+                        for call in calls {
+                            let function_name = &call.function.name;
+                            let arguments = call.function.arguments.clone().unwrap_or_default();
+
+                            let tool_call_msg = HarmonyMessage {
+                                author: Author {
+                                    role: Role::Assistant,
+                                    name: name.clone(),
+                                },
+                                recipient: Some(format!("functions.{}", function_name)),
+                                content: vec![Content::Text(TextContent { text: arguments })],
+                                channel: Some("commentary".to_string()),
+                                content_type: Some("json".to_string()),
+                            };
+                            harmony_messages.push(tool_call_msg);
+                        }
+                    } else {
+                        // Regular assistant message with content
+                        // Combine content with reasoning if present
+                        let mut text = content
+                            .as_ref()
+                            .map(|c| c.to_simple_string())
+                            .unwrap_or_default();
+
+                        if let Some(reasoning) = reasoning_content {
+                            if !text.is_empty() {
+                                text.push('\n');
+                            }
+                            text.push_str(reasoning);
+                        }
+
+                        let harmony_msg = HarmonyMessage {
+                            author: Author {
+                                role: Role::Assistant,
+                                name: name.clone(),
+                            },
+                            recipient: None,
+                            content: vec![Content::Text(TextContent { text })],
+                            channel: Some("final".to_string()),
+                            content_type: None,
+                        };
+                        harmony_messages.push(harmony_msg);
+                    }
+                }
+
+                ChatMessage::Tool {
+                    content,
+                    tool_call_id,
+                } => {
+                    // Look up the function name from the tool_call_id
+                    let function_name = tool_call_map
+                        .get(tool_call_id)
+                        .cloned()
+                        .unwrap_or_else(|| tool_call_id.clone());
+
+                    // Tool result - Must include recipient="assistant" for parser to recognize it.
+                    // We keep channel=None to minimize what the model might copy.
+                    let harmony_msg = HarmonyMessage {
+                        author: Author {
+                            role: Role::Tool,
+                            name: Some(format!("functions.{}", function_name)),
+                        },
+                        recipient: Some("assistant".to_string()),
+                        content: vec![Content::Text(TextContent {
+                            text: content.to_simple_string(),
+                        })],
+                        channel: None,
+                        content_type: None,
+                    };
+                    harmony_messages.push(harmony_msg);
+                }
+
+                ChatMessage::Function { content, name } => {
+                    // Function messages also use Role::Tool
+                    // Tool result - Must include recipient="assistant" for parser to recognize it.
+                    // We keep channel=None to minimize what the model might copy.
+                    let harmony_msg = HarmonyMessage {
+                        author: Author {
+                            role: Role::Tool,
+                            name: Some(format!("functions.{}", name)),
+                        },
+                        recipient: Some("assistant".to_string()),
+                        content: vec![Content::Text(TextContent {
+                            text: content.clone(),
+                        })],
+                        channel: None,
+                        content_type: None,
+                    };
+                    harmony_messages.push(harmony_msg);
+                }
+            }
+        }
+
+        Ok(harmony_messages)
+    }
+
+    /// Extract selection text for worker routing
+    ///
+    /// Uses the last user message for load balancing
+    fn extract_selection_text(&self, messages: &[HarmonyMessage]) -> String {
+        // Find the last user message
+        if let Some(last_user_msg) = messages.iter().rev().find(|m| m.author.role == Role::User) {
+            // Extract full text from content
+            return last_user_msg
+                .content
+                .iter()
+                .filter_map(|c| match c {
+                    Content::Text(tc) => Some(tc.text.as_str()),
+                    _ => None,
+                })
+                .collect::<Vec<_>>()
+                .join("");
+        }
+
+        // Fallback: concatenate all text
+        messages
+            .iter()
+            .flat_map(|m| &m.content)
+            .filter_map(|c| match c {
+                Content::Text(tc) => Some(tc.text.as_str()),
+                _ => None,
+            })
+            .collect::<Vec<_>>()
+            .join(" ")
+    }
+}
+
+impl Default for HarmonyBuilder {
+    fn default() -> Self {
+        Self::new()
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/detector.rs

@@ -0,0 +1,77 @@
+//! Harmony model detection
+
+use crate::core::{Worker, WorkerRegistry};
+
+/// Harmony model detector
+///
+/// Detects if a model name indicates support for Harmony encoding/parsing.
+pub(crate) struct HarmonyDetector;
+
+impl HarmonyDetector {
+    /// Check if a worker is a Harmony/GPT-OSS model.
+    ///
+    /// Detection priority:
+    /// 1. Check if any model card has architectures containing "GptOssForCausalLM"
+    /// 2. Check if any model card has hf_model_type equal to "gpt_oss"
+    /// 3. Check if model_id contains "gpt-oss" substring (case-insensitive)
+    pub fn is_harmony_worker(worker: &dyn Worker) -> bool {
+        for model_card in worker.models() {
+            // 1. Check architectures for GptOssForCausalLM
+            if model_card
+                .architectures
+                .iter()
+                .any(|arch| arch == "GptOssForCausalLM")
+            {
+                return true;
+            }
+
+            // 2. Check hf_model_type for gpt_oss
+            if let Some(ref model_type) = model_card.hf_model_type {
+                if model_type == "gpt_oss" {
+                    return true;
+                }
+            }
+
+            // 3. Check model id for gpt-oss substring
+            if Self::is_harmony_model(&model_card.id) {
+                return true;
+            }
+        }
+
+        // Fallback: check worker's model_id directly
+        Self::is_harmony_model(worker.model_id())
+    }
+
+    /// Check if a model name contains "gpt-oss" (case-insensitive).
+    pub fn is_harmony_model(model_name: &str) -> bool {
+        // Case-insensitive substring search without heap allocation
+        // More efficient than to_lowercase() which allocates a new String
+        model_name
+            .as_bytes()
+            .windows(7) // "gpt-oss".len()
+            .any(|window| window.eq_ignore_ascii_case(b"gpt-oss"))
+    }
+
+    /// Check if any worker for the given model is a Harmony/GPT-OSS worker.
+    ///
+    /// This method looks up workers from the registry by model name and checks
+    /// if any of them are Harmony workers based on their metadata (architectures,
+    /// hf_model_type).
+    ///
+    /// Falls back to string-based detection if no workers are registered for
+    /// the model (e.g., during startup before workers are discovered).
+    pub fn is_harmony_model_in_registry(registry: &WorkerRegistry, model_name: &str) -> bool {
+        // Get workers for this model
+        let workers = registry.get_by_model(model_name);
+
+        if workers.is_empty() {
+            // No workers found - fall back to string-based detection
+            return Self::is_harmony_model(model_name);
+        }
+
+        // Check if any worker is a Harmony worker
+        workers
+            .iter()
+            .any(|worker| Self::is_harmony_worker(worker.as_ref()))
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/mod.rs

@@ -0,0 +1,48 @@
+//! Harmony pipeline implementation
+//!
+//! This module provides support for GPT-OSS models that use Harmony encoding/parsing.
+//! The Harmony protocol uses a channel-based approach with three channels:
+//! - **analysis**: Reasoning/thinking content (optional)
+//! - **commentary**: Tool calls (optional)
+//! - **final**: Final response text (required)
+//!
+//! ## Architecture
+//!
+//! The Harmony implementation is structured as follows:
+//!
+//! - **detector**: Model detection (is this a Harmony-capable model?)
+//! - **builder**: Request encoding (convert Chat/Responses → input_ids)
+//! - **parser**: Response parsing (output_ids → channels)
+//! - **types**: Shared type definitions
+//!
+//! ## Usage
+//!
+//! ```ignore
+//! use smg::routers::grpc::harmony::{HarmonyDetector, HarmonyBuilder};
+//!
+//! // Detect if model supports Harmony
+//! if HarmonyDetector::is_harmony_model("gpt-4o") {
+//!     // Build Harmony request
+//!     let builder = HarmonyBuilder::new();
+//!     let output = builder.build_from_chat(&request)?;
+//!     // ... use output.input_ids for gRPC request
+//! }
+//! ```
+
+pub(crate) mod builder;
+pub(crate) mod detector;
+pub(crate) mod parser;
+pub(crate) mod processor;
+pub(crate) mod responses;
+pub(crate) mod stages;
+pub(crate) mod streaming;
+pub(crate) mod types;
+
+// Re-export types that are accessed via harmony::TypeName
+pub(crate) use builder::HarmonyBuilder;
+pub(crate) use detector::HarmonyDetector;
+pub(crate) use parser::HarmonyParserAdapter;
+pub(crate) use processor::{HarmonyResponseProcessor, ResponsesIterationResult};
+pub(crate) use responses::{serve_harmony_responses, serve_harmony_responses_stream};
+pub(crate) use streaming::HarmonyStreamingProcessor;
+pub(crate) use types::HarmonyMessage;

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/parser.rs

@@ -0,0 +1,536 @@
+//! Harmony response parser
+//!
+//! Adapter for openai_harmony::StreamableParser that handles channel-based parsing.
+
+use openai_harmony::{chat::Role, HarmonyEncoding, StreamableParser};
+use uuid::Uuid;
+
+use super::types::{HarmonyChannelDelta, HarmonyChannelOutput};
+use crate::protocols::common::{FunctionCallResponse, ToolCall};
+
+/// Get the global Harmony encoding
+///
+/// References the same encoding used by the builder for consistency
+fn get_harmony_encoding() -> &'static HarmonyEncoding {
+    use super::builder::get_harmony_encoding;
+    get_harmony_encoding()
+}
+
+/// Harmony parser adapter
+///
+/// Wraps openai_harmony::StreamableParser and provides methods for parsing
+/// complete responses and streaming chunks.
+pub(crate) struct HarmonyParserAdapter {
+    parser: StreamableParser,
+    prev_recipient: Option<String>,
+    reasoning_token_count: u32,
+}
+
+impl HarmonyParserAdapter {
+    /// Create a new Harmony parser
+    pub fn new() -> Result<Self, String> {
+        let encoding = get_harmony_encoding();
+        let parser = StreamableParser::new(encoding.clone(), Some(Role::Assistant))
+            .map_err(|e| format!("Failed to create StreamableParser: {}", e))?;
+
+        Ok(Self {
+            parser,
+            prev_recipient: None,
+            reasoning_token_count: 0,
+        })
+    }
+
+    /// Extract text from message content (private helper)
+    ///
+    /// Filters text content from a message's content array and joins them into a single string.
+    ///
+    /// # Arguments
+    ///
+    /// * `content` - The content array from a Harmony message
+    ///
+    /// # Returns
+    ///
+    /// Joined text string from all text content items
+    fn extract_text_from_content(content: &[openai_harmony::chat::Content]) -> String {
+        content
+            .iter()
+            .filter_map(|c| match c {
+                openai_harmony::chat::Content::Text(tc) => Some(tc.text.as_str()),
+                _ => None,
+            })
+            .collect::<Vec<_>>()
+            .join("")
+    }
+
+    /// Handle incomplete content from parser state (private helper)
+    ///
+    /// Checks for any remaining incomplete content in the parser and appends it
+    /// to the appropriate channel (analysis or final_text).
+    ///
+    /// # Arguments
+    ///
+    /// * `parser` - Reference to the StreamableParser
+    /// * `analysis` - Mutable reference to analysis content
+    /// * `final_text` - Mutable reference to final text content
+    fn handle_incomplete_content(
+        parser: &StreamableParser,
+        analysis: &mut Option<String>,
+        final_text: &mut String,
+    ) {
+        if let Ok(current_content) = parser.current_content() {
+            if !current_content.is_empty() {
+                let current_channel = parser.current_channel();
+                match current_channel.as_deref() {
+                    Some("analysis") => {
+                        *analysis = Some(current_content);
+                    }
+                    Some("final") | None => {
+                        final_text.push_str(&current_content);
+                    }
+                    _ => {}
+                }
+            }
+        }
+    }
+
+    /// Parse messages into channel outputs (private helper)
+    ///
+    /// Extracts analysis, commentary (tool calls), and final text from Harmony messages.
+    /// This is the core parsing logic shared by both parse_complete and finalize.
+    ///
+    /// # Arguments
+    ///
+    /// * `messages` - The messages to parse from the Harmony parser
+    ///
+    /// # Returns
+    ///
+    /// Tuple of (analysis, commentary, final_text)
+    pub fn parse_messages(
+        messages: &[openai_harmony::chat::Message],
+    ) -> (Option<String>, Option<Vec<ToolCall>>, String) {
+        let mut analysis: Option<String> = None;
+        let mut commentary: Option<Vec<ToolCall>> = None;
+        let mut final_text = String::new();
+
+        for msg in messages {
+            // Filter: Only process assistant messages
+            if msg.author.role != Role::Assistant {
+                continue;
+            }
+
+            let channel = msg.channel.as_deref().unwrap_or("");
+            let recipient = msg.recipient.as_deref();
+
+            // IMPORTANT: Check recipient FIRST before channel
+            // The model sometimes generates tool calls with channel="analysis" + recipient="functions.*"
+            // instead of channel="commentary" + recipient="functions.*"
+            // We should trust the recipient field to determine if this is a tool call
+            if let Some(recipient_str) = recipient {
+                if recipient_str.starts_with("functions.") {
+                    // This is a tool call, regardless of channel
+                    let function_name = recipient_str.strip_prefix("functions.").unwrap();
+
+                    // Process each content item separately
+                    for content in &msg.content {
+                        if let openai_harmony::chat::Content::Text(tc) = content {
+                            let call_id = format!("call_{}", Uuid::new_v4());
+                            let tool_call = ToolCall {
+                                id: call_id,
+                                tool_type: "function".to_string(),
+                                function: FunctionCallResponse {
+                                    name: function_name.to_string(),
+                                    arguments: Some(tc.text.clone()),
+                                },
+                            };
+
+                            match commentary.as_mut() {
+                                Some(calls) => calls.push(tool_call),
+                                None => commentary = Some(vec![tool_call]),
+                            }
+                        }
+                    }
+                    // Skip further channel processing for this message
+                    continue;
+                } else if recipient_str.starts_with("python")
+                    || recipient_str.starts_with("browser")
+                    || recipient_str.starts_with("container")
+                {
+                    // Built-in tools → treat as reasoning
+                    // For Chat API, we add to analysis content
+                    let text = Self::extract_text_from_content(&msg.content);
+
+                    if !text.is_empty() {
+                        // Append to analysis (built-in tools are reasoning)
+                        match analysis.as_mut() {
+                            Some(existing) => {
+                                existing.push('\n');
+                                existing.push_str(&text);
+                            }
+                            None => analysis = Some(text),
+                        }
+                    }
+                    // Skip further channel processing
+                    continue;
+                }
+            }
+
+            // Now process by channel (only if not already handled by recipient)
+            match channel {
+                "analysis" => {
+                    // Process each content item
+                    // For Chat API, we join them into a single reasoning_content
+                    let text = Self::extract_text_from_content(&msg.content);
+
+                    if !text.is_empty() {
+                        analysis = Some(text);
+                    }
+                }
+                "commentary" => {
+                    // If we reach here, recipient was not "functions.*" or built-in tools
+                    // Commentary channel should always have a recipient
+                    // This is likely a model bug - log warning and treat as reasoning
+                    tracing::warn!(
+                        channel = "commentary",
+                        recipient = ?recipient,
+                        "Commentary message without valid recipient, treating as reasoning"
+                    );
+
+                    let text = Self::extract_text_from_content(&msg.content);
+
+                    if !text.is_empty() {
+                        match analysis.as_mut() {
+                            Some(existing) => {
+                                existing.push('\n');
+                                existing.push_str(&text);
+                            }
+                            None => analysis = Some(text),
+                        }
+                    }
+                }
+                "final" => {
+                    // Process final channel content
+                    let text = Self::extract_text_from_content(&msg.content);
+                    final_text.push_str(&text);
+                }
+                _ => {
+                    // Unknown channel, append to final text as fallback
+                    let text = Self::extract_text_from_content(&msg.content);
+                    final_text.push_str(&text);
+                }
+            }
+        }
+
+        (analysis, commentary, final_text)
+    }
+
+    /// Parse complete response
+    ///
+    /// Parses all output token IDs and returns the complete channel output
+    /// containing analysis, commentary (tool calls), and final text.
+    ///
+    /// # Arguments
+    ///
+    /// * `output_ids` - The complete output token IDs from the model
+    /// * `finish_reason` - The finish reason from GenerateComplete ("stop", "length", etc.)
+    /// * `matched_stop` - Optional matched stop token information from GenerateComplete
+    ///
+    /// # Returns
+    ///
+    /// Complete HarmonyChannelOutput with all three channels parsed
+    pub fn parse_complete(
+        &mut self,
+        output_ids: &[u32],
+        finish_reason: String,
+        matched_stop: Option<serde_json::Value>,
+    ) -> Result<HarmonyChannelOutput, String> {
+        let mut reasoning_token_count = 0u32;
+
+        // Feed all tokens to the parser
+        for &token_id in output_ids {
+            self.parser
+                .process(token_id)
+                .map_err(|e| format!("Failed to process token {}: {}", token_id, e))?;
+
+            // Count reasoning tokens (analysis + commentary channels)
+            if let Some(channel) = self.parser.current_channel() {
+                if channel == "analysis" || channel == "commentary" {
+                    reasoning_token_count += 1;
+                }
+            }
+        }
+
+        // Extract all completed messages from the parser
+        let messages = self.parser.messages();
+
+        // Parse messages into channel outputs using shared helper
+        let (mut analysis, commentary, mut final_text) = Self::parse_messages(messages);
+
+        // Check for incomplete content in parser state
+        Self::handle_incomplete_content(&self.parser, &mut analysis, &mut final_text);
+
+        // Determine finish reason: override to "tool_calls" if commentary has tool calls
+        let final_finish_reason = if commentary.is_some() {
+            "tool_calls".to_string()
+        } else {
+            finish_reason.clone()
+        };
+
+        Ok(HarmonyChannelOutput {
+            analysis,
+            commentary,
+            final_text,
+            finish_reason: final_finish_reason,
+            matched_stop,
+            reasoning_token_count,
+        })
+    }
+
+    /// Get all messages from the parser
+    ///
+    /// Returns the raw messages extracted by the Harmony parser.
+    /// Used for validation checks.
+    pub fn get_messages(&self) -> Vec<openai_harmony::chat::Message> {
+        self.parser.messages().to_vec()
+    }
+
+    /// Extract incomplete commentary content from parser state
+    ///
+    /// When the stream ends, there may be incomplete commentary content in the parser
+    /// that hasn't been finalized into a completed message. This method extracts
+    /// such content and converts it to tool calls.
+    ///
+    /// # Returns
+    ///
+    /// Optional vector of ToolCall if incomplete commentary is found
+    pub fn extract_incomplete_commentary(&self) -> Option<Vec<ToolCall>> {
+        // Check if current channel is commentary
+        let current_channel = self.parser.current_channel();
+        if current_channel.as_deref() != Some("commentary") {
+            return None;
+        }
+
+        // Get current recipient (should be "functions.{name}")
+        let recipient = self.parser.current_recipient()?;
+        if !recipient.starts_with("functions.") {
+            return None;
+        }
+
+        // Get current incomplete content
+        let content = self.parser.current_content().ok()?;
+        if content.is_empty() {
+            return None;
+        }
+
+        // Extract function name from recipient
+        let function_name = recipient.strip_prefix("functions.").unwrap();
+
+        // Create tool call from incomplete content
+        let call_id = format!("call_{}", Uuid::new_v4());
+        let tool_call = ToolCall {
+            id: call_id,
+            tool_type: "function".to_string(),
+            function: FunctionCallResponse {
+                name: function_name.to_string(),
+                arguments: Some(content),
+            },
+        };
+
+        Some(vec![tool_call])
+    }
+
+    /// Parse streaming chunk
+    ///
+    /// Parses incremental token IDs and returns a delta with any new content
+    /// from the analysis, commentary, or final channels.
+    ///
+    /// # Arguments
+    ///
+    /// * `chunk_ids` - New token IDs from the current chunk
+    ///
+    /// # Returns
+    ///
+    /// Optional HarmonyChannelDelta if there's new content to emit
+    pub fn parse_chunk(
+        &mut self,
+        chunk_ids: &[u32],
+    ) -> Result<Option<HarmonyChannelDelta>, String> {
+        let mut has_delta = false;
+        let mut analysis_delta = None;
+        let mut commentary_delta = None;
+        let mut final_delta = None;
+
+        // Track message count before processing
+        let prev_message_count = self.parser.messages().len();
+
+        // Accumulate delta text for commentary channel
+        let mut accumulated_delta = String::new();
+
+        // Process each token
+        for &token_id in chunk_ids {
+            self.parser
+                .process(token_id)
+                .map_err(|e| format!("Failed to process token {}: {}", token_id, e))?;
+
+            // Count reasoning tokens (analysis + commentary channels)
+            if let Some(channel) = self.parser.current_channel() {
+                if channel == "analysis" || channel == "commentary" {
+                    self.reasoning_token_count += 1;
+                }
+            }
+
+            // Check for content delta
+            if let Ok(Some(delta_text)) = self.parser.last_content_delta() {
+                has_delta = true;
+
+                // Determine which channel this delta belongs to
+                let channel = self.parser.current_channel();
+                match channel.as_deref() {
+                    Some("analysis") => {
+                        analysis_delta = Some(delta_text);
+                    }
+                    Some("final") | None => {
+                        final_delta = Some(delta_text);
+                    }
+                    Some("commentary") => {
+                        // Accumulate delta for commentary
+                        accumulated_delta.push_str(&delta_text);
+                    }
+                    _ => {}
+                }
+            }
+        }
+
+        // Handle commentary channel tool call deltas
+        if self.parser.current_channel().as_deref() == Some("commentary") {
+            if let Some(cur_recipient) = self.parser.current_recipient() {
+                if cur_recipient.starts_with("functions.") {
+                    has_delta = true;
+
+                    // Count completed tool calls for index
+                    let base_index = self
+                        .parser
+                        .messages()
+                        .iter()
+                        .filter(|msg| {
+                            msg.channel.as_deref() == Some("commentary")
+                                && msg
+                                    .recipient
+                                    .as_deref()
+                                    .is_some_and(|r| r.starts_with("functions."))
+                        })
+                        .count();
+
+                    // Check if recipient changed (new tool call)
+                    let recipient_changed = self.prev_recipient.as_deref() != Some(&cur_recipient);
+
+                    if recipient_changed {
+                        // NEW tool call: emit name + id
+                        let tool_name = cur_recipient.strip_prefix("functions.").unwrap();
+                        let call_id = format!("call_{}", Uuid::new_v4());
+
+                        commentary_delta = Some(super::types::ToolCallDelta {
+                            index: base_index,
+                            id: Some(call_id),
+                            function: Some(super::types::FunctionDelta {
+                                name: Some(tool_name.to_string()),
+                                arguments: Some(String::new()),
+                            }),
+                        });
+
+                        // Update prev_recipient
+                        self.prev_recipient = Some(cur_recipient);
+                    } else if !accumulated_delta.is_empty() {
+                        // CONTINUING tool call: emit arguments delta
+                        commentary_delta = Some(super::types::ToolCallDelta {
+                            index: base_index,
+                            id: None,
+                            function: Some(super::types::FunctionDelta {
+                                name: None,
+                                arguments: Some(accumulated_delta),
+                            }),
+                        });
+                    }
+                }
+            }
+        }
+
+        // Check if new messages were completed
+        let current_message_count = self.parser.messages().len();
+        let is_final = current_message_count > prev_message_count;
+
+        if has_delta {
+            Ok(Some(HarmonyChannelDelta {
+                analysis_delta,
+                commentary_delta,
+                final_delta,
+                is_final,
+            }))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Finalize parsing
+    ///
+    /// Called at the end of streaming to get the final state and any
+    /// remaining content.
+    ///
+    /// # Arguments
+    ///
+    /// * `finish_reason` - The finish reason from GenerateComplete ("stop", "length", etc.)
+    /// * `matched_stop` - Optional matched stop token information from GenerateComplete
+    ///
+    /// # Returns
+    ///
+    /// Final HarmonyChannelOutput with complete parsed content
+    pub fn finalize(
+        &mut self,
+        finish_reason: String,
+        matched_stop: Option<serde_json::Value>,
+    ) -> Result<HarmonyChannelOutput, String> {
+        // Extract all completed messages
+        let messages = self.parser.messages();
+
+        // Parse messages into channel outputs using shared helper
+        let (mut analysis, commentary, mut final_text) = Self::parse_messages(messages);
+
+        // Check for remaining incomplete content
+        Self::handle_incomplete_content(&self.parser, &mut analysis, &mut final_text);
+
+        // Determine finish reason: override to "tool_calls" if commentary has tool calls
+        let final_finish_reason = if commentary.is_some() {
+            "tool_calls".to_string()
+        } else {
+            finish_reason
+        };
+
+        Ok(HarmonyChannelOutput {
+            analysis,
+            commentary,
+            final_text,
+            finish_reason: final_finish_reason,
+            matched_stop,
+            reasoning_token_count: self.reasoning_token_count,
+        })
+    }
+
+    /// Reset parser state
+    ///
+    /// Resets the parser to initial state for reuse
+    #[allow(dead_code)]
+    pub fn reset(&mut self) -> Result<(), String> {
+        // Create a new parser instance (StreamableParser doesn't have a reset method)
+        let encoding = get_harmony_encoding();
+        self.parser = StreamableParser::new(encoding.clone(), Some(Role::Assistant))
+            .map_err(|e| format!("Failed to reset parser: {}", e))?;
+        self.prev_recipient = None;
+        self.reasoning_token_count = 0;
+        Ok(())
+    }
+}
+
+impl Default for HarmonyParserAdapter {
+    fn default() -> Self {
+        Self::new().expect("Failed to create default parser")
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/processor.rs

@@ -0,0 +1,345 @@
+//! Harmony response processor for non-streaming responses
+
+use std::sync::Arc;
+
+use axum::response::Response;
+use smg_grpc_client::sglang_proto::generate_complete::MatchedStop::{
+    MatchedStopStr, MatchedTokenId,
+};
+use tracing::error;
+
+use super::HarmonyParserAdapter;
+use crate::{
+    protocols::{
+        chat::{ChatChoice, ChatCompletionMessage, ChatCompletionRequest, ChatCompletionResponse},
+        common::{CompletionTokensDetails, ToolCall, Usage},
+        responses::{
+            OutputTokensDetails, ResponseContentPart, ResponseOutputItem, ResponseReasoningContent,
+            ResponseStatus, ResponseUsage, ResponsesRequest, ResponsesResponse, ResponsesUsage,
+        },
+    },
+    routers::{
+        error,
+        grpc::{
+            common::{response_collection, response_formatting},
+            context::{DispatchMetadata, ExecutionResult},
+        },
+    },
+};
+
+/// Processor for non-streaming Harmony responses
+///
+/// Collects all output tokens from execution and parses them using
+/// HarmonyParserAdapter to extract the complete response.
+pub(crate) struct HarmonyResponseProcessor;
+
+impl HarmonyResponseProcessor {
+    /// Create a new Harmony response processor
+    pub fn new() -> Self {
+        Self
+    }
+
+    /// Process a non-streaming Harmony chat response
+    pub async fn process_non_streaming_chat_response(
+        &self,
+        execution_result: ExecutionResult,
+        chat_request: Arc<ChatCompletionRequest>,
+        dispatch: DispatchMetadata,
+    ) -> Result<ChatCompletionResponse, Response> {
+        // Collect all completed responses (one per choice)
+        let all_responses = response_collection::collect_responses(execution_result, false).await?;
+        if all_responses.is_empty() {
+            return Err(error::internal_error(
+                "no_responses_from_server",
+                "No responses from server",
+            ));
+        }
+
+        // Build choices by parsing output with HarmonyParserAdapter
+        let mut choices: Vec<ChatChoice> = Vec::new();
+        let mut total_reasoning_tokens = 0u32;
+
+        for (index, complete) in all_responses.iter().enumerate() {
+            // Convert matched_stop from proto to JSON
+            let matched_stop = complete.matched_stop().map(|m| match m {
+                MatchedTokenId(id) => {
+                    serde_json::json!(id)
+                }
+                MatchedStopStr(s) => {
+                    serde_json::json!(s)
+                }
+            });
+
+            // Parse Harmony channels with HarmonyParserAdapter
+            let mut parser = HarmonyParserAdapter::new().map_err(|e| {
+                error!(
+                    function = "process_non_streaming_chat_response",
+                    error = %e,
+                    "Failed to create Harmony parser"
+                );
+                error::internal_error(
+                    "create_harmony_parser_failed",
+                    format!("Failed to create Harmony parser: {}", e),
+                )
+            })?;
+
+            // Parse Harmony channels with finish_reason and matched_stop
+            let parsed = parser
+                .parse_complete(
+                    complete.output_ids(),
+                    complete.finish_reason().to_string(),
+                    matched_stop.clone(),
+                )
+                .map_err(|e| {
+                    error!(
+                        function = "process_non_streaming_chat_response",
+                        error = %e,
+                        "Harmony parsing failed on complete response"
+                    );
+                    error::internal_error(
+                        "harmony_parsing_failed",
+                        format!("Harmony parsing failed: {}", e),
+                    )
+                })?;
+
+            // Build response message (assistant)
+            let message = ChatCompletionMessage {
+                role: "assistant".to_string(),
+                content: (!parsed.final_text.is_empty()).then_some(parsed.final_text),
+                tool_calls: parsed.commentary,
+                reasoning_content: parsed.analysis,
+            };
+
+            let finish_reason = parsed.finish_reason;
+
+            // Accumulate reasoning tokens across all responses
+            total_reasoning_tokens += parsed.reasoning_token_count;
+
+            choices.push(ChatChoice {
+                index: index as u32,
+                message,
+                logprobs: None,
+                finish_reason: Some(finish_reason),
+                matched_stop,
+                hidden_states: None,
+            });
+        }
+
+        // Build usage from proto fields
+        let mut usage = response_formatting::build_usage(&all_responses);
+
+        // Add reasoning token count from parsed analysis/commentary channels
+        if total_reasoning_tokens > 0 {
+            usage.completion_tokens_details = Some(CompletionTokensDetails {
+                reasoning_tokens: Some(total_reasoning_tokens),
+            });
+        }
+
+        // Final ChatCompletionResponse
+        Ok(
+            ChatCompletionResponse::builder(&dispatch.request_id, &chat_request.model)
+                .created(dispatch.created)
+                .choices(choices)
+                .usage(usage)
+                .maybe_system_fingerprint(dispatch.weight_version.as_deref())
+                .build(),
+        )
+    }
+}
+
+impl Default for HarmonyResponseProcessor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Result of processing a single Responses API iteration
+///
+/// Used by the MCP tool loop to determine whether to continue
+/// executing tools or return the final response.
+pub(crate) enum ResponsesIterationResult {
+    /// Tool calls found in commentary channel - continue MCP loop
+    ToolCallsFound {
+        tool_calls: Vec<ToolCall>,
+        analysis: Option<String>, // For streaming emission or reasoning output
+        partial_text: String,     // For streaming emission or message output
+        usage: Usage,             // Token usage from this iteration
+        request_id: String,       // Request ID from dispatch
+    },
+    /// No tool calls - return final ResponsesResponse
+    Completed {
+        response: Box<ResponsesResponse>,
+        usage: Usage,
+    },
+}
+
+impl HarmonyResponseProcessor {
+    /// Process a single Responses API iteration
+    ///
+    /// Parses Harmony channels and determines if tool calls are present.
+    /// If tool calls found, returns ToolCallsFound for MCP loop to execute.
+    /// If no tool calls, builds final ResponsesResponse.
+    ///
+    /// # Arguments
+    ///
+    /// * `execution_result` - The execution result from the model
+    /// * `responses_request` - The original Responses API request
+    /// * `dispatch` - Dispatch metadata for request tracking
+    ///
+    /// # Returns
+    ///
+    /// ResponsesIterationResult indicating whether to continue loop or return
+    pub async fn process_responses_iteration(
+        &self,
+        execution_result: ExecutionResult,
+        responses_request: Arc<ResponsesRequest>,
+        dispatch: DispatchMetadata,
+    ) -> Result<ResponsesIterationResult, Response> {
+        // Collect all completed responses
+        let all_responses = response_collection::collect_responses(execution_result, false).await?;
+        if all_responses.is_empty() {
+            return Err(error::internal_error(
+                "no_responses_from_server",
+                "No responses from server",
+            ));
+        }
+
+        // For Responses API, we only process the first response (n=1)
+        let complete = all_responses
+            .first()
+            .ok_or_else(|| error::internal_error("no_complete_response", "No complete response"))?;
+
+        // Parse Harmony channels
+        let mut parser = HarmonyParserAdapter::new().map_err(|e| {
+            error!(
+                function = "process_responses_iteration",
+                error = %e,
+                "Failed to create Harmony parser"
+            );
+            error::internal_error(
+                "create_harmony_parser_failed",
+                format!("Failed to create Harmony parser: {}", e),
+            )
+        })?;
+
+        // Convert matched_stop from proto to JSON
+        let matched_stop = complete.matched_stop().map(|m| match m {
+            MatchedTokenId(id) => {
+                serde_json::json!(id)
+            }
+            MatchedStopStr(s) => {
+                serde_json::json!(s)
+            }
+        });
+
+        let parsed = parser
+            .parse_complete(
+                complete.output_ids(),
+                complete.finish_reason().to_string(),
+                matched_stop,
+            )
+            .map_err(|e| {
+                error!(
+                    function = "process_responses_iteration",
+                    error = %e,
+                    "Harmony parsing failed on complete response"
+                );
+                error::internal_error(
+                    "harmony_parsing_failed",
+                    format!("Harmony parsing failed: {}", e),
+                )
+            })?;
+
+        // VALIDATION: Check if model incorrectly generated Tool role messages
+        // This happens when the model copies the format of tool result messages
+        // instead of continuing as assistant. This is a model hallucination bug.
+        let messages = parser.get_messages();
+        let tool_messages_generated = messages.iter().any(|msg| {
+            msg.author.role == openai_harmony::chat::Role::Tool
+                && msg.recipient.as_deref() == Some("assistant")
+        });
+
+        if tool_messages_generated {
+            tracing::warn!(
+                "Model generated Tool->Assistant message instead of Assistant message. \
+                This is a model hallucination bug where it copies tool result format."
+            );
+        }
+
+        // Build usage (needed for both ToolCallsFound and Completed)
+        let mut usage = response_formatting::build_usage(std::slice::from_ref(complete));
+
+        // Add reasoning token count from parsed analysis/commentary channels
+        if parsed.reasoning_token_count > 0 {
+            usage.completion_tokens_details = Some(CompletionTokensDetails {
+                reasoning_tokens: Some(parsed.reasoning_token_count),
+            });
+        }
+
+        // Check for tool calls in commentary channel
+        if let Some(tool_calls) = parsed.commentary {
+            // Tool calls found - return for MCP loop execution
+            return Ok(ResponsesIterationResult::ToolCallsFound {
+                tool_calls,
+                analysis: parsed.analysis,
+                partial_text: parsed.final_text,
+                usage,
+                request_id: dispatch.request_id.clone(),
+            });
+        }
+
+        // No tool calls - build final ResponsesResponse
+        let mut output: Vec<ResponseOutputItem> = Vec::new();
+
+        // Map analysis channel → ResponseOutputItem::Reasoning
+        if let Some(analysis) = parsed.analysis {
+            let reasoning_item = ResponseOutputItem::Reasoning {
+                id: format!("reasoning_{}", dispatch.request_id),
+                summary: vec![],
+                content: vec![ResponseReasoningContent::ReasoningText { text: analysis }],
+                status: Some("completed".to_string()),
+            };
+            output.push(reasoning_item);
+        }
+
+        // Map final channel → ResponseOutputItem::Message
+        if !parsed.final_text.is_empty() {
+            let message_item = ResponseOutputItem::Message {
+                id: format!("msg_{}", dispatch.request_id),
+                role: "assistant".to_string(),
+                content: vec![ResponseContentPart::OutputText {
+                    text: parsed.final_text,
+                    annotations: vec![],
+                    logprobs: None,
+                }],
+                status: "completed".to_string(),
+            };
+            output.push(message_item);
+        }
+
+        // Build ResponsesResponse with all required fields
+        let response = ResponsesResponse::builder(&dispatch.request_id, &responses_request.model)
+            .copy_from_request(&responses_request)
+            .created_at(dispatch.created as i64)
+            .status(ResponseStatus::Completed)
+            .output(output)
+            .maybe_text(responses_request.text.clone())
+            .usage(ResponsesUsage::Modern(ResponseUsage {
+                input_tokens: usage.prompt_tokens,
+                output_tokens: usage.completion_tokens,
+                total_tokens: usage.total_tokens,
+                input_tokens_details: None,
+                output_tokens_details: usage.completion_tokens_details.as_ref().and_then(|d| {
+                    d.reasoning_tokens.map(|tokens| OutputTokensDetails {
+                        reasoning_tokens: tokens,
+                    })
+                }),
+            }))
+            .build();
+
+        Ok(ResponsesIterationResult::Completed {
+            response: Box::new(response),
+            usage,
+        })
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/responses/common.rs

@@ -0,0 +1,364 @@
+//! Shared helpers and state tracking for Harmony Responses
+
+use axum::response::Response;
+use data_connector::ResponseId;
+use serde_json::{from_value, json, to_string, Value};
+use smg_mcp as mcp;
+use tracing::{debug, error, warn};
+use uuid::Uuid;
+
+use super::execution::ToolResult;
+use crate::{
+    protocols::{
+        common::{ToolCall, ToolChoice, ToolChoiceValue},
+        responses::{
+            McpToolInfo, ResponseContentPart, ResponseInput, ResponseInputOutputItem,
+            ResponseOutputItem, ResponseReasoningContent, ResponseTool, ResponseToolType,
+            ResponsesRequest, ResponsesResponse, StringOrContentParts,
+        },
+    },
+    routers::{error, grpc::common::responses::ResponsesContext},
+};
+
+/// Record of a single MCP tool call execution
+///
+/// Stores metadata needed to build mcp_call output items for Responses API format
+#[derive(Debug, Clone)]
+pub(super) struct McpCallRecord {
+    /// Tool call ID (stored for potential future use, currently generate new IDs)
+    #[allow(dead_code)]
+    pub call_id: String,
+    /// Tool name
+    pub tool_name: String,
+    /// JSON-encoded arguments
+    pub arguments: String,
+    /// JSON-encoded output/result
+    pub output: String,
+    /// Whether execution succeeded
+    pub success: bool,
+    /// Error message if execution failed
+    pub error: Option<String>,
+}
+
+/// Tracking structure for MCP tool calls across iterations
+///
+/// Accumulates all MCP tool call metadata during multi-turn conversation
+/// so we can build proper mcp_list_tools and mcp_call output items.
+#[derive(Debug, Clone)]
+pub(super) struct McpCallTracking {
+    /// MCP server label (e.g., "sglang-mcp")
+    pub server_label: String,
+    /// All tool call records across all iterations
+    pub tool_calls: Vec<McpCallRecord>,
+}
+
+impl McpCallTracking {
+    pub fn new(server_label: String) -> Self {
+        Self {
+            server_label,
+            tool_calls: Vec::new(),
+        }
+    }
+
+    pub fn record_call(
+        &mut self,
+        call_id: String,
+        tool_name: String,
+        arguments: String,
+        output: String,
+        success: bool,
+        error: Option<String>,
+    ) {
+        self.tool_calls.push(McpCallRecord {
+            call_id,
+            tool_name,
+            arguments,
+            output,
+            success,
+            error,
+        });
+    }
+
+    pub fn total_calls(&self) -> usize {
+        self.tool_calls.len()
+    }
+}
+
+/// Build a HashSet of MCP tool names for O(1) lookup
+///
+/// Creates a HashSet containing the names of all MCP tools in the request,
+/// allowing for efficient O(1) lookups when partitioning tool calls.
+pub(super) fn build_mcp_tool_names_set(
+    request_tools: &[ResponseTool],
+) -> std::collections::HashSet<&str> {
+    request_tools
+        .iter()
+        .filter(|t| t.r#type == ResponseToolType::Mcp)
+        .filter_map(|t| t.function.as_ref().map(|f| f.name.as_str()))
+        .collect()
+}
+
+/// Build next request with tool results appended to history
+///
+/// Constructs a new ResponsesRequest with:
+/// 1. Original input items (preserved)
+/// 2. Assistant message with analysis (reasoning) + partial_text + tool_calls
+/// 3. Tool result messages for each tool execution
+pub(super) fn build_next_request_with_tools(
+    mut request: ResponsesRequest,
+    tool_calls: Vec<ToolCall>,
+    tool_results: Vec<ToolResult>,
+    analysis: Option<String>, // Analysis channel content (becomes reasoning content)
+    partial_text: String,     // Final channel content (becomes message content)
+) -> Result<ResponsesRequest, Box<Response>> {
+    // Get current input items (or empty vec if Text variant)
+    let mut items = match request.input {
+        ResponseInput::Items(items) => items,
+        ResponseInput::Text(text) => {
+            // Convert text to items format
+            vec![ResponseInputOutputItem::SimpleInputMessage {
+                content: StringOrContentParts::String(text),
+                role: "user".to_string(),
+                r#type: None,
+            }]
+        }
+    };
+
+    // Build assistant response item with reasoning + content + tool calls
+    // This represents what the model generated in this iteration
+    let assistant_id = format!("msg_{}", Uuid::new_v4());
+
+    // Add reasoning if present (from analysis channel)
+    if let Some(analysis_text) = analysis {
+        items.push(ResponseInputOutputItem::Reasoning {
+            id: format!("reasoning_{}", assistant_id),
+            summary: vec![],
+            content: vec![ResponseReasoningContent::ReasoningText {
+                text: analysis_text,
+            }],
+            status: Some("completed".to_string()),
+        });
+    }
+
+    // Add message content if present (from final channel)
+    if !partial_text.is_empty() {
+        items.push(ResponseInputOutputItem::Message {
+            id: assistant_id.clone(),
+            role: "assistant".to_string(),
+            content: vec![ResponseContentPart::OutputText {
+                text: partial_text,
+                annotations: vec![],
+                logprobs: None,
+            }],
+            status: Some("completed".to_string()),
+        });
+    }
+
+    // Add function tool calls (from commentary channel)
+    for tool_call in tool_calls {
+        items.push(ResponseInputOutputItem::FunctionToolCall {
+            id: tool_call.id.clone(),
+            call_id: tool_call.id.clone(),
+            name: tool_call.function.name.clone(),
+            arguments: tool_call
+                .function
+                .arguments
+                .unwrap_or_else(|| "{}".to_string()),
+            output: None, // Output will be added next
+            status: Some("in_progress".to_string()),
+        });
+    }
+
+    // Add tool results
+    for tool_result in tool_results {
+        // Serialize tool output to string
+        let output_str = to_string(&tool_result.output).unwrap_or_else(|e| {
+            format!("{{\"error\": \"Failed to serialize tool output: {}\"}}", e)
+        });
+
+        // Update the corresponding tool call with output and completed status
+        // Find and update the matching FunctionToolCall
+        if let Some(ResponseInputOutputItem::FunctionToolCall {
+            output,
+            status,
+            ..
+        }) = items
+            .iter_mut()
+            .find(|item| matches!(item, ResponseInputOutputItem::FunctionToolCall { call_id, .. } if call_id == &tool_result.call_id))
+        {
+            *output = Some(output_str);
+            *status = if tool_result.is_error {
+                Some("failed".to_string())
+            } else {
+                Some("completed".to_string())
+            };
+        }
+    }
+
+    // Update request with new items
+    request.input = ResponseInput::Items(items);
+
+    // Switch tool_choice to "auto" for subsequent iterations
+    // This prevents infinite loops when original tool_choice was "required" or specific function
+    // After receiving tool results, the model should be free to decide whether to call more tools or finish
+    request.tool_choice = Some(ToolChoice::Value(ToolChoiceValue::Auto));
+
+    Ok(request)
+}
+
+/// Inject MCP metadata into final response
+///
+/// Adds mcp_list_tools and mcp_call output items to the response output array.
+/// Following non-Harmony pipeline pattern:
+/// 1. Prepend mcp_list_tools at the beginning
+/// 2. Append all mcp_call items at the end
+pub(super) fn inject_mcp_metadata(
+    response: &mut ResponsesResponse,
+    tracking: &McpCallTracking,
+    mcp_tools: &[mcp::Tool],
+) {
+    // Build mcp_list_tools item
+    let tools = mcp_tools;
+    let tools_info: Vec<McpToolInfo> = tools
+        .iter()
+        .map(|t| McpToolInfo {
+            name: t.name.to_string(),
+            description: t.description.as_ref().map(|d| d.to_string()),
+            input_schema: Value::Object((*t.input_schema).clone()),
+            annotations: Some(json!({
+                "read_only": false
+            })),
+        })
+        .collect();
+
+    let mcp_list_tools = ResponseOutputItem::McpListTools {
+        id: format!("mcpl_{}", Uuid::new_v4()),
+        server_label: tracking.server_label.clone(),
+        tools: tools_info,
+    };
+
+    // Build mcp_call items for each tracked call
+    let mcp_call_items: Vec<ResponseOutputItem> = tracking
+        .tool_calls
+        .iter()
+        .map(|record| ResponseOutputItem::McpCall {
+            id: format!("mcp_{}", Uuid::new_v4()),
+            status: if record.success {
+                "completed"
+            } else {
+                "failed"
+            }
+            .to_string(),
+            approval_request_id: None,
+            arguments: record.arguments.clone(),
+            error: record.error.clone(),
+            name: record.tool_name.clone(),
+            output: record.output.clone(),
+            server_label: tracking.server_label.clone(),
+        })
+        .collect();
+
+    // Inject into response output:
+    // 1. Prepend mcp_list_tools at the beginning
+    response.output.insert(0, mcp_list_tools);
+
+    // 2. Append all mcp_call items at the end
+    response.output.extend(mcp_call_items);
+}
+
+/// Load previous conversation messages from storage
+///
+/// If the request has `previous_response_id`, loads the response chain from storage
+/// and prepends the conversation history to the request input items.
+pub(super) async fn load_previous_messages(
+    ctx: &ResponsesContext,
+    request: ResponsesRequest,
+) -> Result<ResponsesRequest, Response> {
+    let Some(ref prev_id_str) = request.previous_response_id else {
+        // No previous_response_id, return request as-is
+        return Ok(request);
+    };
+
+    let prev_id = ResponseId::from(prev_id_str.as_str());
+
+    // Load response chain from storage
+    let chain = ctx
+        .response_storage
+        .get_response_chain(&prev_id, None)
+        .await
+        .map_err(|e| {
+            error!(
+                function = "load_previous_messages",
+                prev_id = %prev_id_str,
+                error = %e,
+                "Failed to load previous response chain from storage"
+            );
+            error::internal_error(
+                "load_previous_response_chain_failed",
+                format!(
+                    "Failed to load previous response chain for {}: {}",
+                    prev_id_str, e
+                ),
+            )
+        })?;
+
+    // Build conversation history from stored responses
+    let mut history_items = Vec::new();
+
+    // Helper to deserialize and collect items from a JSON array
+    let deserialize_items = |arr: &Value, item_type: &str| -> Vec<ResponseInputOutputItem> {
+        arr.as_array()
+            .into_iter()
+            .flat_map(|items| items.iter())
+            .filter_map(|item| {
+                from_value::<ResponseInputOutputItem>(item.clone())
+                    .map_err(|e| {
+                        warn!(
+                            "Failed to deserialize stored {} item: {}. Item: {}",
+                            item_type, e, item
+                        );
+                    })
+                    .ok()
+            })
+            .collect()
+    };
+
+    for stored in chain.responses.iter() {
+        history_items.extend(deserialize_items(&stored.input, "input"));
+        history_items.extend(deserialize_items(&stored.output, "output"));
+    }
+
+    debug!(
+        previous_response_id = %prev_id_str,
+        history_items_count = history_items.len(),
+        "Loaded conversation history from previous response"
+    );
+
+    // Build modified request with history prepended
+    let mut modified_request = request;
+
+    // Convert current input to items format
+    let all_items = match modified_request.input {
+        ResponseInput::Items(items) => {
+            // Prepend history to existing items
+            let mut combined = history_items;
+            combined.extend(items);
+            combined
+        }
+        ResponseInput::Text(text) => {
+            // Convert text to item and prepend history
+            history_items.push(ResponseInputOutputItem::SimpleInputMessage {
+                content: StringOrContentParts::String(text),
+                role: "user".to_string(),
+                r#type: None,
+            });
+            history_items
+        }
+    };
+
+    // Update request with combined items and clear previous_response_id
+    modified_request.input = ResponseInput::Items(all_items);
+    modified_request.previous_response_id = None;
+
+    Ok(modified_request)
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/responses/execution.rs

@@ -0,0 +1,224 @@
+//! MCP tool execution logic for Harmony Responses
+
+use std::{sync::Arc, time::Instant};
+
+use axum::response::Response;
+use serde_json::{from_str, json, to_string, to_value, Value};
+use smg_mcp::{self as mcp, McpManager};
+use tracing::{debug, error, warn};
+
+use super::common::McpCallTracking;
+use crate::{
+    observability::metrics::{metrics_labels, Metrics},
+    protocols::{
+        common::{Function, ToolCall},
+        responses::{ResponseTool, ResponseToolType},
+    },
+    routers::error,
+};
+
+/// Tool execution result
+///
+/// Contains the result of executing a single MCP tool.
+pub(crate) struct ToolResult {
+    /// Tool call ID (for matching with request)
+    pub call_id: String,
+
+    /// Tool name
+    #[allow(dead_code)] // Kept for documentation and future use
+    pub tool_name: String,
+
+    /// Tool output (JSON value)
+    pub output: Value,
+
+    /// Whether this is an error result
+    pub is_error: bool,
+}
+
+/// Execute MCP tools and collect results
+///
+/// Executes each tool call sequentially via the MCP manager.
+/// Tool execution errors are returned as error results to the model
+/// (allows model to handle gracefully).
+///
+/// Vector of tool results (one per tool call)
+pub(super) async fn execute_mcp_tools(
+    mcp_manager: &Arc<McpManager>,
+    tool_calls: &[ToolCall],
+    tracking: &mut McpCallTracking,
+    model_id: &str,
+) -> Result<Vec<ToolResult>, Response> {
+    let mut results = Vec::new();
+
+    for tool_call in tool_calls {
+        debug!(
+            tool_name = %tool_call.function.name,
+            call_id = %tool_call.id,
+            "Executing MCP tool"
+        );
+
+        // Parse tool arguments from JSON string
+        let args_str = tool_call.function.arguments.as_deref().unwrap_or("{}");
+        let args: Value = from_str(args_str).map_err(|e| {
+            error!(
+                function = "execute_mcp_tools",
+                tool_name = %tool_call.function.name,
+                call_id = %tool_call.id,
+                error = %e,
+                "Failed to parse tool arguments JSON"
+            );
+            error::internal_error(
+                "invalid_tool_args",
+                format!(
+                    "Invalid tool arguments JSON for tool '{}': {}",
+                    tool_call.function.name, e
+                ),
+            )
+        })?;
+
+        // Execute tool via MCP manager
+        let args_map = if let Value::Object(map) = args {
+            Some(map)
+        } else {
+            None
+        };
+
+        let tool_start = Instant::now();
+        let tool_result = mcp_manager
+            .call_tool(&tool_call.function.name, args_map)
+            .await;
+        let tool_duration = tool_start.elapsed();
+
+        match tool_result {
+            Ok(mcp_result) => {
+                debug!(
+                    tool_name = %tool_call.function.name,
+                    call_id = %tool_call.id,
+                    "Tool execution succeeded"
+                );
+
+                // Extract content from MCP result
+                let output = if let Some(content) = mcp_result.content.first() {
+                    // Serialize the entire content item
+                    to_value(content)
+                        .unwrap_or_else(|_| json!({"error": "Failed to serialize tool result"}))
+                } else {
+                    json!({"result": "success"})
+                };
+
+                let is_error = mcp_result.is_error.unwrap_or(false);
+                let output_str = to_string(&output)
+                    .unwrap_or_else(|_| r#"{"error": "Failed to serialize output"}"#.to_string());
+
+                // Record this call in tracking
+                tracking.record_call(
+                    tool_call.id.clone(),
+                    tool_call.function.name.clone(),
+                    args_str.to_string(),
+                    output_str.clone(),
+                    !is_error,
+                    if is_error {
+                        Some(output_str.clone())
+                    } else {
+                        None
+                    },
+                );
+
+                // Record MCP tool metrics
+                Metrics::record_mcp_tool_duration(
+                    model_id,
+                    &tool_call.function.name,
+                    tool_duration,
+                );
+                Metrics::record_mcp_tool_call(
+                    model_id,
+                    &tool_call.function.name,
+                    if is_error {
+                        metrics_labels::RESULT_ERROR
+                    } else {
+                        metrics_labels::RESULT_SUCCESS
+                    },
+                );
+
+                results.push(ToolResult {
+                    call_id: tool_call.id.clone(),
+                    tool_name: tool_call.function.name.clone(),
+                    output,
+                    is_error,
+                });
+            }
+            Err(e) => {
+                warn!(
+                    tool_name = %tool_call.function.name,
+                    call_id = %tool_call.id,
+                    error = %e,
+                    "Tool execution failed"
+                );
+
+                let error_msg = format!("Tool execution failed: {}", e);
+                let error_output = json!({
+                    "error": error_msg.clone()
+                });
+                let error_output_str = to_string(&error_output)
+                    .unwrap_or_else(|_| format!(r#"{{"error": "{}"}}"#, error_msg));
+
+                // Record failed call in tracking
+                tracking.record_call(
+                    tool_call.id.clone(),
+                    tool_call.function.name.clone(),
+                    args_str.to_string(),
+                    error_output_str.clone(),
+                    false,
+                    Some(error_msg),
+                );
+
+                // Record MCP tool metrics
+                Metrics::record_mcp_tool_duration(
+                    model_id,
+                    &tool_call.function.name,
+                    tool_duration,
+                );
+                Metrics::record_mcp_tool_call(
+                    model_id,
+                    &tool_call.function.name,
+                    metrics_labels::RESULT_ERROR,
+                );
+
+                // Return error result to model (let it handle gracefully)
+                results.push(ToolResult {
+                    call_id: tool_call.id.clone(),
+                    tool_name: tool_call.function.name.clone(),
+                    output: error_output,
+                    is_error: true,
+                });
+            }
+        }
+    }
+
+    Ok(results)
+}
+
+/// Convert MCP tools to Responses API tool format
+///
+/// Converts MCP Tool entries (from rmcp SDK) to ResponseTool format so the model
+/// knows about available MCP tools when making tool calls.
+pub(crate) fn convert_mcp_tools_to_response_tools(mcp_tools: &[mcp::Tool]) -> Vec<ResponseTool> {
+    mcp_tools
+        .iter()
+        .map(|tool_info| ResponseTool {
+            r#type: ResponseToolType::Mcp,
+            function: Some(Function {
+                name: tool_info.name.to_string(),
+                description: tool_info.description.as_ref().map(|d| d.to_string()),
+                parameters: Value::Object((*tool_info.input_schema).clone()),
+                strict: None,
+            }),
+            server_url: None, // MCP tools from inventory don't have individual server URLs
+            authorization: None,
+            server_label: None,
+            server_description: tool_info.description.as_ref().map(|d| d.to_string()),
+            require_approval: None,
+            allowed_tools: None,
+        })
+        .collect()
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/responses/mod.rs

@@ -0,0 +1,29 @@
+//! Harmony Responses API implementation with multi-turn MCP tool support
+//!
+//! This module implements the Harmony Responses API orchestration logic,
+//! coordinating full pipeline execution with MCP tool support for multi-turn conversations.
+//!
+//! ## Architecture
+//!
+//! Multi-turn pipeline orchestration (NOT just a tool loop):
+//! - Serves Harmony Responses API requests end-to-end
+//! - Each iteration executes FULL pipeline (worker selection + client acquisition + execution + parsing)
+//! - Handles MCP tool execution and history building between iterations
+//! - Clean separation: serving orchestration vs. pipeline stages (stages/)
+//!
+//! ## Module Structure
+//!
+//! - `non_streaming` - Non-streaming entry point and tool loop
+//! - `streaming` - Streaming entry point and tool loop
+//! - `execution` - MCP tool execution logic
+//! - `common` - Shared helpers and state tracking
+
+pub(crate) mod common;
+pub(crate) mod execution;
+pub(crate) mod non_streaming;
+pub(crate) mod streaming;
+
+// Re-export types accessed via harmony::responses::TypeName
+pub(crate) use execution::ToolResult;
+pub(crate) use non_streaming::serve_harmony_responses;
+pub(crate) use streaming::serve_harmony_responses_stream;

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/responses/non_streaming.rs

@@ -0,0 +1,463 @@
+//! Non-streaming Harmony Responses API implementation
+
+use std::{
+    sync::Arc,
+    time::{SystemTime, UNIX_EPOCH},
+};
+
+use axum::response::Response;
+use serde_json::{json, to_string};
+use tracing::{debug, error, warn};
+
+use super::{
+    common::{
+        build_mcp_tool_names_set, build_next_request_with_tools, inject_mcp_metadata,
+        load_previous_messages, McpCallTracking,
+    },
+    execution::{convert_mcp_tools_to_response_tools, execute_mcp_tools, ToolResult},
+};
+use crate::{
+    observability::metrics::Metrics,
+    protocols::{
+        common::{ToolCall, Usage},
+        responses::{
+            OutputTokensDetails, ResponseContentPart, ResponseOutputItem, ResponseReasoningContent,
+            ResponseStatus, ResponseUsage, ResponsesRequest, ResponsesResponse, ResponsesUsage,
+        },
+    },
+    routers::{
+        error,
+        grpc::{
+            common::responses::{
+                ensure_mcp_connection, persist_response_if_needed, ResponsesContext,
+            },
+            harmony::processor::ResponsesIterationResult,
+        },
+        mcp_utils::{extract_server_label, DEFAULT_MAX_ITERATIONS},
+    },
+};
+
+/// Execute Harmony Responses API request with multi-turn MCP tool support
+///
+/// This function orchestrates the multi-turn conversation flow:
+/// 1. Execute request through full pipeline
+/// 2. Check for tool calls in commentary channel
+/// 3. If tool calls found:
+///    - Execute MCP tools
+///    - Build next request with tool results
+///    - Repeat from step 1 (full pipeline re-execution)
+/// 4. If no tool calls, return final response
+pub(crate) async fn serve_harmony_responses(
+    ctx: &ResponsesContext,
+    request: ResponsesRequest,
+) -> Result<ResponsesResponse, Response> {
+    // Clone request for persistence
+    let original_request = request.clone();
+
+    // Load previous conversation history if previous_response_id is set
+    let current_request = load_previous_messages(ctx, request).await?;
+
+    // Check MCP connection and get whether MCP tools are present
+    let (has_mcp_tools, server_keys) =
+        ensure_mcp_connection(&ctx.mcp_manager, current_request.tools.as_deref()).await?;
+
+    // Set the server keys in the context
+    {
+        let mut servers = ctx.requested_servers.write().unwrap();
+        *servers = server_keys;
+    }
+
+    let response = if has_mcp_tools {
+        execute_with_mcp_loop(ctx, current_request).await?
+    } else {
+        // No MCP tools - execute pipeline once (may have function tools or no tools)
+        execute_without_mcp_loop(ctx, current_request).await?
+    };
+
+    // Persist response to storage if store=true
+    persist_response_if_needed(
+        ctx.conversation_storage.clone(),
+        ctx.conversation_item_storage.clone(),
+        ctx.response_storage.clone(),
+        &response,
+        &original_request,
+    )
+    .await;
+
+    Ok(response)
+}
+
+/// Execute Harmony Responses with MCP tool loop
+///
+/// Automatically executes MCP tools in a loop until no more tool calls or max iterations
+async fn execute_with_mcp_loop(
+    ctx: &ResponsesContext,
+    mut current_request: ResponsesRequest,
+) -> Result<ResponsesResponse, Response> {
+    let mut iteration_count = 0;
+
+    // Extract server_label from request tools
+    let server_label = extract_server_label(current_request.tools.as_deref(), "sglang-mcp");
+    let mut mcp_tracking = McpCallTracking::new(server_label.clone());
+
+    // Extract user's max_tool_calls limit (if set)
+    let max_tool_calls = current_request.max_tool_calls.map(|n| n as usize);
+
+    // Add filtered MCP tools (static + requested dynamic) to the request
+    let mcp_tools = {
+        let servers = ctx.requested_servers.read().unwrap();
+        ctx.mcp_manager.list_tools_for_servers(&servers)
+    };
+    if !mcp_tools.is_empty() {
+        let mcp_response_tools = convert_mcp_tools_to_response_tools(&mcp_tools);
+
+        let mut all_tools = current_request.tools.clone().unwrap_or_default();
+        all_tools.extend(mcp_response_tools);
+        current_request.tools = Some(all_tools);
+
+        debug!(
+            mcp_tool_count = mcp_tools.len(),
+            total_tool_count = current_request.tools.as_ref().map(|t| t.len()).unwrap_or(0),
+            "MCP client available - added static MCP tools to Harmony Responses request"
+        );
+    }
+
+    loop {
+        iteration_count += 1;
+
+        // Record tool loop iteration metric
+        Metrics::record_mcp_tool_iteration(&current_request.model);
+
+        // Safety check: prevent infinite loops
+        if iteration_count > DEFAULT_MAX_ITERATIONS {
+            error!(
+                function = "execute_with_mcp_loop",
+                iteration_count = iteration_count,
+                max_iterations = DEFAULT_MAX_ITERATIONS,
+                "Maximum tool iterations exceeded"
+            );
+            return Err(error::internal_error(
+                "tool_iterations_exceeded",
+                format!(
+                    "Maximum tool iterations ({}) exceeded",
+                    DEFAULT_MAX_ITERATIONS
+                ),
+            ));
+        }
+
+        debug!(
+            iteration = iteration_count,
+            "Harmony Responses serving iteration"
+        );
+
+        // Execute through full pipeline
+        let iteration_result = ctx
+            .pipeline
+            .execute_harmony_responses(&current_request, ctx)
+            .await?;
+
+        match iteration_result {
+            ResponsesIterationResult::ToolCallsFound {
+                tool_calls,
+                analysis,
+                partial_text,
+                usage,
+                request_id,
+            } => {
+                debug!(
+                    tool_call_count = tool_calls.len(),
+                    has_analysis = analysis.is_some(),
+                    partial_text_len = partial_text.len(),
+                    "Tool calls found - separating MCP and function tools"
+                );
+
+                // Separate MCP and function tool calls based on tool type
+                let request_tools = current_request.tools.as_deref().unwrap_or(&[]);
+                let mcp_tool_names = build_mcp_tool_names_set(request_tools);
+                let (mcp_tool_calls, function_tool_calls): (Vec<_>, Vec<_>) = tool_calls
+                    .into_iter()
+                    .partition(|tc| mcp_tool_names.contains(tc.function.name.as_str()));
+
+                debug!(
+                    mcp_calls = mcp_tool_calls.len(),
+                    function_calls = function_tool_calls.len(),
+                    "Tool calls separated by type"
+                );
+
+                // Check combined limit (user's max_tool_calls vs safety limit)
+                let effective_limit = match max_tool_calls {
+                    Some(user_max) => user_max.min(DEFAULT_MAX_ITERATIONS),
+                    None => DEFAULT_MAX_ITERATIONS,
+                };
+
+                // Check if we would exceed the limit with these new MCP tool calls
+                let total_calls_after = mcp_tracking.total_calls() + mcp_tool_calls.len();
+                if total_calls_after > effective_limit {
+                    warn!(
+                        current_calls = mcp_tracking.total_calls(),
+                        new_calls = mcp_tool_calls.len() + function_tool_calls.len(),
+                        total_after = total_calls_after,
+                        effective_limit = effective_limit,
+                        user_max = ?max_tool_calls,
+                        "Reached tool call limit - returning incomplete response"
+                    );
+
+                    // Combine back for response
+                    let all_tool_calls: Vec<_> = mcp_tool_calls
+                        .into_iter()
+                        .chain(function_tool_calls)
+                        .collect();
+
+                    // Build response with incomplete status - no tools executed due to limit
+                    let mut response = build_tool_response(
+                        vec![],         // No MCP tools executed
+                        vec![],         // No MCP results
+                        all_tool_calls, // All tools returned as function calls (not executed)
+                        analysis,
+                        partial_text,
+                        usage,
+                        request_id,
+                        Arc::new(current_request),
+                    );
+
+                    // Mark as completed with incomplete_details
+                    response.status = ResponseStatus::Completed;
+                    response.incomplete_details = Some(json!({ "reason": "max_tool_calls" }));
+
+                    // Inject MCP metadata if any calls were executed
+                    if mcp_tracking.total_calls() > 0 {
+                        inject_mcp_metadata(&mut response, &mcp_tracking, &mcp_tools);
+                    }
+
+                    return Ok(response);
+                }
+
+                // Execute MCP tools (if any)
+                let mcp_results = if !mcp_tool_calls.is_empty() {
+                    execute_mcp_tools(
+                        &ctx.mcp_manager,
+                        &mcp_tool_calls,
+                        &mut mcp_tracking,
+                        &current_request.model,
+                    )
+                    .await?
+                } else {
+                    Vec::new()
+                };
+
+                // If there are function tools, exit MCP loop and return response
+                if !function_tool_calls.is_empty() {
+                    debug!(
+                        "Function tool calls present - exiting MCP loop and returning to caller"
+                    );
+
+                    // Build response that includes:
+                    // 1. Reasoning/message from this iteration
+                    // 2. MCP tools as completed (with output) - these were executed
+                    // 3. Function tools as completed (without output) - need caller execution
+                    let mut response = build_tool_response(
+                        mcp_tool_calls,
+                        mcp_results,
+                        function_tool_calls,
+                        analysis,
+                        partial_text,
+                        usage,
+                        request_id,
+                        Arc::new(current_request),
+                    );
+
+                    // Inject MCP metadata for all executed calls
+                    if mcp_tracking.total_calls() > 0 {
+                        inject_mcp_metadata(&mut response, &mcp_tracking, &mcp_tools);
+                    }
+
+                    return Ok(response);
+                }
+
+                // Only MCP tools - continue loop with their results
+                debug!("Only MCP tools - continuing loop with results");
+
+                // Build next request with appended history
+                current_request = build_next_request_with_tools(
+                    current_request,
+                    mcp_tool_calls,
+                    mcp_results,
+                    analysis,
+                    partial_text,
+                )
+                .map_err(|e| *e)?;
+
+                // Continue loop - next iteration will select workers and execute
+            }
+            ResponsesIterationResult::Completed {
+                mut response,
+                usage,
+            } => {
+                debug!(
+                    output_items = response.output.len(),
+                    input_tokens = usage.prompt_tokens,
+                    output_tokens = usage.completion_tokens,
+                    "MCP loop completed - no more tool calls"
+                );
+
+                // Inject MCP metadata into final response
+                inject_mcp_metadata(&mut response, &mcp_tracking, &mcp_tools);
+
+                debug!(
+                    mcp_calls = mcp_tracking.total_calls(),
+                    output_items_after = response.output.len(),
+                    "Injected MCP metadata into final response"
+                );
+
+                // No tool calls - this is the final response
+                return Ok(*response);
+            }
+        }
+    }
+}
+
+/// Execute Harmony Responses without MCP loop (single execution)
+///
+/// For function tools or no tools - executes pipeline once and returns
+async fn execute_without_mcp_loop(
+    ctx: &ResponsesContext,
+    current_request: ResponsesRequest,
+) -> Result<ResponsesResponse, Response> {
+    debug!("Executing Harmony Responses without MCP loop");
+
+    // Execute pipeline once
+    let iteration_result = ctx
+        .pipeline
+        .execute_harmony_responses(&current_request, ctx)
+        .await?;
+
+    match iteration_result {
+        ResponsesIterationResult::ToolCallsFound {
+            tool_calls,
+            analysis,
+            partial_text,
+            usage,
+            request_id,
+        } => {
+            // Function tool calls found - return to caller for execution
+            debug!(
+                tool_call_count = tool_calls.len(),
+                "Function tool calls found - returning to caller"
+            );
+
+            Ok(build_tool_response(
+                vec![],
+                vec![],
+                tool_calls,
+                analysis,
+                partial_text,
+                usage,
+                request_id,
+                Arc::new(current_request),
+            ))
+        }
+        ResponsesIterationResult::Completed { response, usage: _ } => {
+            // No tool calls - return completed response
+            debug!("No tool calls - returning completed response");
+            Ok(*response)
+        }
+    }
+}
+
+/// Build ResponsesResponse with tool calls (MCP and/or function tools)
+#[allow(clippy::too_many_arguments)]
+fn build_tool_response(
+    mcp_tool_calls: Vec<ToolCall>,
+    mcp_results: Vec<ToolResult>,
+    function_tool_calls: Vec<ToolCall>,
+    analysis: Option<String>, // Analysis channel content (reasoning)
+    partial_text: String,     // Final channel content (message)
+    usage: Usage,
+    request_id: String,
+    responses_request: Arc<ResponsesRequest>,
+) -> ResponsesResponse {
+    let mut output: Vec<ResponseOutputItem> = Vec::new();
+
+    // Add reasoning output item if analysis exists
+    if let Some(analysis_text) = analysis {
+        output.push(ResponseOutputItem::Reasoning {
+            id: format!("reasoning_{}", request_id),
+            summary: vec![],
+            content: vec![ResponseReasoningContent::ReasoningText {
+                text: analysis_text,
+            }],
+            status: Some("completed".to_string()),
+        });
+    }
+
+    // Add message output item if partial text exists
+    if !partial_text.is_empty() {
+        output.push(ResponseOutputItem::Message {
+            id: format!("msg_{}", request_id),
+            role: "assistant".to_string(),
+            content: vec![ResponseContentPart::OutputText {
+                text: partial_text,
+                annotations: vec![],
+                logprobs: None,
+            }],
+            status: "completed".to_string(),
+        });
+    }
+
+    // Add MCP tool calls WITH output (these were executed)
+    for (tool_call, result) in mcp_tool_calls.iter().zip(mcp_results.iter()) {
+        let output_str = to_string(&result.output).unwrap_or_else(|e| {
+            format!("{{\"error\": \"Failed to serialize tool output: {}\"}}", e)
+        });
+
+        output.push(ResponseOutputItem::FunctionToolCall {
+            id: tool_call.id.clone(),
+            call_id: tool_call.id.clone(),
+            name: tool_call.function.name.clone(),
+            arguments: tool_call.function.arguments.clone().unwrap_or_default(),
+            output: Some(output_str),
+            status: if result.is_error {
+                "failed"
+            } else {
+                "completed"
+            }
+            .to_string(),
+        });
+    }
+
+    // Add function tool calls WITHOUT output (need caller execution)
+    for tool_call in function_tool_calls {
+        output.push(ResponseOutputItem::FunctionToolCall {
+            id: tool_call.id.clone(),
+            call_id: tool_call.id.clone(),
+            name: tool_call.function.name.clone(),
+            arguments: tool_call.function.arguments.clone().unwrap_or_default(),
+            output: None, // No output = needs execution
+            status: "completed".to_string(),
+        });
+    }
+
+    // Build ResponsesResponse with Completed status
+    let created_at = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs() as i64;
+
+    ResponsesResponse::builder(&request_id, &responses_request.model)
+        .copy_from_request(&responses_request)
+        .created_at(created_at)
+        .status(ResponseStatus::Completed)
+        .output(output)
+        .usage(ResponsesUsage::Modern(ResponseUsage {
+            input_tokens: usage.prompt_tokens,
+            output_tokens: usage.completion_tokens,
+            total_tokens: usage.total_tokens,
+            input_tokens_details: None,
+            output_tokens_details: usage.completion_tokens_details.as_ref().and_then(|d| {
+                d.reasoning_tokens.map(|tokens| OutputTokensDetails {
+                    reasoning_tokens: tokens,
+                })
+            }),
+        }))
+        .build()
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/responses/streaming.rs

@@ -0,0 +1,534 @@
+//! Streaming Harmony Responses API implementation
+
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use axum::response::Response;
+use bytes::Bytes;
+use serde_json::{json, Value};
+use tokio::sync::mpsc;
+use tracing::{debug, warn};
+use uuid::Uuid;
+
+use super::{
+    common::{
+        build_mcp_tool_names_set, build_next_request_with_tools, load_previous_messages,
+        McpCallTracking,
+    },
+    execution::{convert_mcp_tools_to_response_tools, execute_mcp_tools},
+};
+use crate::{
+    observability::metrics::Metrics,
+    protocols::responses::{ResponseToolType, ResponsesRequest},
+    routers::{
+        grpc::{
+            common::responses::{
+                build_sse_response, ensure_mcp_connection, persist_response_if_needed,
+                streaming::{OutputItemType, ResponseStreamEventEmitter},
+                ResponsesContext,
+            },
+            harmony::{processor::ResponsesIterationResult, streaming::HarmonyStreamingProcessor},
+        },
+        mcp_utils::{extract_server_label, DEFAULT_MAX_ITERATIONS},
+    },
+};
+
+/// Serve Harmony Responses API with streaming (SSE)
+///
+/// This is the streaming equivalent of `serve_harmony_responses()`.
+/// Emits SSE events for lifecycle, MCP list_tools, and per-iteration streaming.
+pub(crate) async fn serve_harmony_responses_stream(
+    ctx: &ResponsesContext,
+    request: ResponsesRequest,
+) -> Response {
+    // Load previous conversation history if previous_response_id is set
+    let current_request = match load_previous_messages(ctx, request.clone()).await {
+        Ok(req) => req,
+        Err(err_response) => return err_response,
+    };
+
+    // Check MCP connection BEFORE starting stream and get whether MCP tools are present
+    let (has_mcp_tools, server_keys) =
+        match ensure_mcp_connection(&ctx.mcp_manager, current_request.tools.as_deref()).await {
+            Ok(result) => result,
+            Err(response) => return response,
+        };
+
+    // Set the server keys in the context
+    {
+        let mut servers = ctx.requested_servers.write().unwrap();
+        *servers = server_keys;
+    }
+
+    // Create SSE channel
+    let (tx, rx) = mpsc::unbounded_channel();
+
+    // Create response event emitter
+    let response_id = format!("resp_{}", Uuid::new_v4());
+    let model = current_request.model.clone();
+    let created_at = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs();
+    let mut emitter = ResponseStreamEventEmitter::new(response_id.clone(), model, created_at);
+
+    // Set original request for complete response fields
+    emitter.set_original_request(current_request.clone());
+
+    // Clone context for spawned task
+    let ctx_clone = ctx.clone();
+
+    // Spawn async task to handle streaming
+    tokio::spawn(async move {
+        let ctx = &ctx_clone;
+
+        // Emit initial response.created and response.in_progress events
+        let event = emitter.emit_created();
+        if emitter.send_event(&event, &tx).is_err() {
+            return;
+        }
+        let event = emitter.emit_in_progress();
+        if emitter.send_event(&event, &tx).is_err() {
+            return;
+        }
+
+        if has_mcp_tools {
+            execute_mcp_tool_loop_streaming(ctx, current_request, &request, &mut emitter, &tx)
+                .await;
+        } else {
+            execute_without_mcp_streaming(ctx, &current_request, &request, &mut emitter, &tx).await;
+        }
+    });
+
+    // Return SSE stream response
+    build_sse_response(rx)
+}
+
+/// Execute MCP tool loop with streaming
+///
+/// Handles the full MCP workflow:
+/// - Adds static MCP tools to request
+/// - Emits mcp_list_tools events
+/// - Loops through tool execution iterations
+/// - Emits final response.completed event
+/// - Persists response internally
+async fn execute_mcp_tool_loop_streaming(
+    ctx: &ResponsesContext,
+    mut current_request: ResponsesRequest,
+    original_request: &ResponsesRequest,
+    emitter: &mut ResponseStreamEventEmitter,
+    tx: &mpsc::UnboundedSender<Result<Bytes, std::io::Error>>,
+) {
+    // Extract server_label from request tools
+    let server_label = extract_server_label(current_request.tools.as_deref(), "sglang-mcp");
+
+    // Set server label in emitter for MCP call items
+    emitter.set_mcp_server_label(server_label.clone());
+
+    // Initialize MCP call tracking
+    let mut mcp_tracking = McpCallTracking::new(server_label.clone());
+
+    // Extract user's max_tool_calls limit (if set)
+    let max_tool_calls = current_request.max_tool_calls.map(|n| n as usize);
+
+    // Add filtered MCP tools (static + requested dynamic) to the request
+    let mcp_tools = {
+        let servers = ctx.requested_servers.read().unwrap();
+        ctx.mcp_manager.list_tools_for_servers(&servers)
+    };
+    if !mcp_tools.is_empty() {
+        let mcp_response_tools = convert_mcp_tools_to_response_tools(&mcp_tools);
+        let mut all_tools = current_request.tools.clone().unwrap_or_default();
+        all_tools.extend(mcp_response_tools);
+        current_request.tools = Some(all_tools);
+
+        debug!(
+            mcp_tool_count = mcp_tools.len(),
+            total_tool_count = current_request.tools.as_ref().map(|t| t.len()).unwrap_or(0),
+            "MCP client available - added static MCP tools to Harmony Responses streaming request"
+        );
+    }
+
+    // Build HashSet of MCP tool names for O(1) lookup during streaming
+    // Clone tool names to owned strings to avoid borrowing current_request
+    let mcp_tool_names: std::collections::HashSet<String> = current_request
+        .tools
+        .as_ref()
+        .map(|tools| {
+            tools
+                .iter()
+                .filter(|t| t.r#type == ResponseToolType::Mcp)
+                .filter_map(|t| t.function.as_ref().map(|f| f.name.clone()))
+                .collect()
+        })
+        .unwrap_or_default();
+
+    // Emit mcp_list_tools on first iteration
+    let (output_index, item_id) = emitter.allocate_output_index(OutputItemType::McpListTools);
+
+    // Build tools list for item structure
+    let tool_items: Vec<_> = mcp_tools
+        .iter()
+        .map(|t| {
+            json!({
+                "name": t.name,
+                "description": t.description,
+                "input_schema": Value::Object((*t.input_schema).clone())
+            })
+        })
+        .collect();
+
+    // Build final item with completed status and tools
+    let item_done = json!({
+        "id": item_id,
+        "type": "mcp_list_tools",
+        "server_label": server_label,
+        "status": "completed",
+        "tools": tool_items
+    });
+
+    // Store the completed item data and mark as completed FIRST
+    // This ensures it appears in final response even if event sending fails
+    emitter.emit_output_item_done(output_index, &item_done);
+    emitter.complete_output_item(output_index);
+
+    // Now emit all the events (failures won't affect the stored data)
+    // Emit output_item.added
+    let item = json!({
+        "id": item_id,
+        "type": "mcp_list_tools",
+        "server_label": server_label,
+        "status": "in_progress",
+        "tools": []
+    });
+    let event = emitter.emit_output_item_added(output_index, &item);
+    if emitter.send_event(&event, tx).is_err() {
+        return;
+    }
+
+    // Emit mcp_list_tools.in_progress
+    let event = emitter.emit_mcp_list_tools_in_progress(output_index);
+    if emitter.send_event(&event, tx).is_err() {
+        return;
+    }
+
+    // Emit mcp_list_tools.completed
+    let event = emitter.emit_mcp_list_tools_completed(output_index, &mcp_tools);
+    if emitter.send_event(&event, tx).is_err() {
+        return;
+    }
+
+    // Emit output_item.done
+    let event = emitter.emit_output_item_done(output_index, &item_done);
+    if emitter.send_event(&event, tx).is_err() {
+        return;
+    }
+
+    debug!(
+        tool_count = mcp_tools.len(),
+        "Emitted mcp_list_tools on first iteration"
+    );
+
+    // MCP tool loop (max 10 iterations)
+    let mut iteration_count = 0;
+    loop {
+        iteration_count += 1;
+
+        // Record tool loop iteration metric
+        Metrics::record_mcp_tool_iteration(&current_request.model);
+
+        // Safety check: prevent infinite loops
+        if iteration_count > DEFAULT_MAX_ITERATIONS {
+            emitter.emit_error(
+                &format!(
+                    "Maximum tool iterations ({}) exceeded",
+                    DEFAULT_MAX_ITERATIONS
+                ),
+                Some("max_iterations_exceeded"),
+                tx,
+            );
+            return;
+        }
+
+        debug!(
+            iteration = iteration_count,
+            "Harmony Responses streaming iteration"
+        );
+
+        // Execute pipeline and get stream + load guards
+        let (execution_result, _load_guards) = match ctx
+            .pipeline
+            .execute_harmony_responses_streaming(&current_request, ctx)
+            .await
+        {
+            Ok(result) => result,
+            Err(err_response) => {
+                emitter.emit_error(
+                    &format!("Pipeline execution failed: {:?}", err_response),
+                    Some("pipeline_error"),
+                    tx,
+                );
+                return;
+            }
+        };
+
+        // Process stream with token-level streaming (mixed tools - emits correct events per tool type)
+        // Load guards are held during processing and dropped when iteration completes
+        let iteration_result = match HarmonyStreamingProcessor::process_responses_iteration_stream(
+            execution_result,
+            emitter,
+            tx,
+            &mcp_tool_names,
+        )
+        .await
+        {
+            Ok(result) => result,
+            Err(err_msg) => {
+                emitter.emit_error(&err_msg, Some("processing_error"), tx);
+                return;
+            }
+        };
+
+        // Handle iteration result (tool calls or completion)
+        match iteration_result {
+            ResponsesIterationResult::ToolCallsFound {
+                tool_calls,
+                analysis,
+                partial_text,
+                usage,
+                request_id: _,
+            } => {
+                debug!(
+                    tool_call_count = tool_calls.len(),
+                    has_analysis = analysis.is_some(),
+                    partial_text_len = partial_text.len(),
+                    "Tool calls found - separating MCP and function tools"
+                );
+
+                // Separate MCP and function tool calls based on tool type
+                let request_tools = current_request.tools.as_deref().unwrap_or(&[]);
+                let mcp_tool_names = build_mcp_tool_names_set(request_tools);
+                let (mcp_tool_calls, function_tool_calls): (Vec<_>, Vec<_>) = tool_calls
+                    .into_iter()
+                    .partition(|tc| mcp_tool_names.contains(tc.function.name.as_str()));
+
+                debug!(
+                    mcp_calls = mcp_tool_calls.len(),
+                    function_calls = function_tool_calls.len(),
+                    "Tool calls separated by type in streaming"
+                );
+
+                // Check combined limit (user's max_tool_calls vs safety limit)
+                let effective_limit = match max_tool_calls {
+                    Some(user_max) => user_max.min(DEFAULT_MAX_ITERATIONS),
+                    None => DEFAULT_MAX_ITERATIONS,
+                };
+
+                // Check if we would exceed the limit with these new MCP tool calls
+                let total_calls_after = mcp_tracking.total_calls() + mcp_tool_calls.len();
+                if total_calls_after > effective_limit {
+                    warn!(
+                        current_calls = mcp_tracking.total_calls(),
+                        new_calls = mcp_tool_calls.len() + function_tool_calls.len(),
+                        total_after = total_calls_after,
+                        effective_limit = effective_limit,
+                        user_max = ?max_tool_calls,
+                        "Reached tool call limit in streaming - emitting completion with incomplete_details"
+                    );
+
+                    // Emit response.completed with incomplete_details and usage
+                    let incomplete_details = json!({ "reason": "max_tool_calls" });
+                    let usage_json = json!({
+                        "input_tokens": usage.prompt_tokens,
+                        "output_tokens": usage.completion_tokens,
+                        "total_tokens": usage.total_tokens,
+                        "incomplete_details": incomplete_details,
+                    });
+                    let event = emitter.emit_completed(Some(&usage_json));
+                    emitter.send_event_best_effort(&event, tx);
+                    return;
+                }
+
+                // Execute MCP tools (if any)
+                let mcp_results = if !mcp_tool_calls.is_empty() {
+                    match execute_mcp_tools(
+                        &ctx.mcp_manager,
+                        &mcp_tool_calls,
+                        &mut mcp_tracking,
+                        &current_request.model,
+                    )
+                    .await
+                    {
+                        Ok(results) => results,
+                        Err(err_response) => {
+                            emitter.emit_error(
+                                &format!("MCP tool execution failed: {:?}", err_response),
+                                Some("mcp_tool_error"),
+                                tx,
+                            );
+                            return;
+                        }
+                    }
+                } else {
+                    Vec::new()
+                };
+
+                // Update mcp_call output items with execution results (if any MCP tools were executed)
+                if !mcp_results.is_empty() {
+                    emitter.update_mcp_call_outputs(&mcp_results);
+                }
+
+                // If there are function tools, exit MCP loop and emit completion
+                if !function_tool_calls.is_empty() {
+                    debug!(
+                        "Function tool calls present - exiting MCP loop and emitting completion"
+                    );
+
+                    // Function tool calls were already emitted during streaming processing
+                    // Just emit response.completed with usage
+                    let usage_json = json!({
+                        "input_tokens": usage.prompt_tokens,
+                        "output_tokens": usage.completion_tokens,
+                        "total_tokens": usage.total_tokens,
+                    });
+                    let event = emitter.emit_completed(Some(&usage_json));
+                    emitter.send_event_best_effort(&event, tx);
+                    return;
+                }
+
+                // Only MCP tools - continue loop with their results
+                debug!("Only MCP tools - continuing loop with results");
+
+                // Build next request with appended history
+                current_request = match build_next_request_with_tools(
+                    current_request,
+                    mcp_tool_calls,
+                    mcp_results,
+                    analysis,
+                    partial_text,
+                ) {
+                    Ok(req) => req,
+                    Err(e) => {
+                        emitter.emit_error(
+                            &format!("Failed to build next request: {:?}", e),
+                            Some("request_building_error"),
+                            tx,
+                        );
+                        return;
+                    }
+                };
+
+                // Continue loop
+            }
+            ResponsesIterationResult::Completed { response, usage } => {
+                debug!(
+                    output_items = response.output.len(),
+                    input_tokens = usage.prompt_tokens,
+                    output_tokens = usage.completion_tokens,
+                    "Harmony Responses streaming completed - no more tool calls"
+                );
+
+                // Finalize response from emitter's accumulated data
+                let final_response = emitter.finalize(Some(usage.clone()));
+
+                // Persist response to storage if store=true
+                persist_response_if_needed(
+                    ctx.conversation_storage.clone(),
+                    ctx.conversation_item_storage.clone(),
+                    ctx.response_storage.clone(),
+                    &final_response,
+                    original_request,
+                )
+                .await;
+
+                // Emit response.completed with usage
+                let usage_json = json!({
+                    "input_tokens": usage.prompt_tokens,
+                    "output_tokens": usage.completion_tokens,
+                    "total_tokens": usage.total_tokens,
+                });
+                let event = emitter.emit_completed(Some(&usage_json));
+                emitter.send_event_best_effort(&event, tx);
+                return;
+            }
+        }
+    }
+}
+
+/// Execute without MCP tool loop (single execution with streaming)
+///
+/// For function tools or no tools - executes pipeline once and emits completion.
+/// The streaming processor handles all output items (reasoning, message, function tool calls).
+async fn execute_without_mcp_streaming(
+    ctx: &ResponsesContext,
+    current_request: &ResponsesRequest,
+    original_request: &ResponsesRequest,
+    emitter: &mut ResponseStreamEventEmitter,
+    tx: &mpsc::UnboundedSender<Result<Bytes, std::io::Error>>,
+) {
+    debug!("No MCP tools - executing single iteration");
+
+    // Execute pipeline and get stream + load guards
+    let (execution_result, _load_guards) = match ctx
+        .pipeline
+        .execute_harmony_responses_streaming(current_request, ctx)
+        .await
+    {
+        Ok(result) => result,
+        Err(err_response) => {
+            emitter.emit_error(
+                &format!("Pipeline execution failed: {:?}", err_response),
+                Some("pipeline_error"),
+                tx,
+            );
+            return;
+        }
+    };
+
+    // Process stream (emits all output items during streaming - function tool path emits function_call_arguments.* events)
+    // Pass empty HashSet so all tools are treated as function tools (per-tool detection)
+    // Load guards are held during processing and dropped when iteration completes
+    let empty_mcp_tools = std::collections::HashSet::new();
+    let iteration_result = match HarmonyStreamingProcessor::process_responses_iteration_stream(
+        execution_result,
+        emitter,
+        tx,
+        &empty_mcp_tools,
+    )
+    .await
+    {
+        Ok(result) => result,
+        Err(err_msg) => {
+            emitter.emit_error(&err_msg, Some("processing_error"), tx);
+            return;
+        }
+    };
+    // _load_guards dropped here after iteration completes
+
+    // Extract usage from iteration result
+    let usage = match iteration_result {
+        ResponsesIterationResult::ToolCallsFound { usage, .. } => usage,
+        ResponsesIterationResult::Completed { usage, .. } => usage,
+    };
+
+    // Finalize response from emitter's accumulated data
+    let final_response = emitter.finalize(Some(usage.clone()));
+
+    // Persist response to storage if store=true
+    persist_response_if_needed(
+        ctx.conversation_storage.clone(),
+        ctx.conversation_item_storage.clone(),
+        ctx.response_storage.clone(),
+        &final_response,
+        original_request,
+    )
+    .await;
+
+    // Emit response.completed with usage
+    let usage_json = json!({
+        "input_tokens": usage.prompt_tokens,
+        "output_tokens": usage.completion_tokens,
+        "total_tokens": usage.total_tokens,
+    });
+    let event = emitter.emit_completed(Some(&usage_json));
+    emitter.send_event_best_effort(&event, tx);
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/stages/mod.rs

@@ -0,0 +1,14 @@
+//! Harmony-specific pipeline stages
+//!
+//! These stages replace their regular counterparts in the Harmony pipeline:
+//! - HarmonyPreparationStage: Harmony encoding instead of chat template + tokenization
+//! - HarmonyRequestBuildingStage: Token-based request building
+//! - HarmonyResponseProcessingStage: Harmony channel parsing
+
+pub(crate) mod preparation;
+pub(crate) mod request_building;
+pub(crate) mod response_processing;
+
+pub(crate) use preparation::HarmonyPreparationStage;
+pub(crate) use request_building::HarmonyRequestBuildingStage;
+pub(crate) use response_processing::HarmonyResponseProcessingStage;

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/stages/preparation.rs

@@ -0,0 +1,428 @@
+//! Harmony Preparation Stage: Harmony encoding for chat and generate requests
+
+use async_trait::async_trait;
+use axum::response::Response;
+use serde_json::json;
+use tracing::error;
+
+use super::super::HarmonyBuilder;
+use crate::{
+    protocols::{
+        chat::ChatCompletionRequest,
+        common::{Tool, ToolChoice, ToolChoiceValue},
+        responses::ResponsesRequest,
+    },
+    routers::{
+        error,
+        grpc::{
+            common::{responses::utils::extract_tools_from_response_tools, stages::PipelineStage},
+            context::{PreparationOutput, RequestContext, RequestType},
+            utils,
+        },
+    },
+};
+
+/// Harmony Preparation stage: Encode requests using Harmony protocol
+///
+/// Replaces the regular PreparationStage for Harmony models.
+/// Converts chat/generate requests to Harmony-encoded token_ids and extraction_text.
+pub(crate) struct HarmonyPreparationStage {
+    builder: HarmonyBuilder,
+}
+
+impl HarmonyPreparationStage {
+    /// Create a new Harmony preparation stage
+    pub fn new() -> Self {
+        Self {
+            builder: HarmonyBuilder::new(),
+        }
+    }
+}
+
+impl Default for HarmonyPreparationStage {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl PipelineStage for HarmonyPreparationStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        // Clone Arc before match to avoid borrow checker issues
+        // Arc clone is cheap (8 bytes) - avoids full request clone (15KB-200KB)
+        let is_chat = matches!(&ctx.input.request_type, RequestType::Chat(_));
+        let is_responses = matches!(&ctx.input.request_type, RequestType::Responses(_));
+
+        if is_chat {
+            let request_arc = ctx.chat_request_arc();
+            self.prepare_chat(ctx, &request_arc).await?;
+        } else if is_responses {
+            let request_arc = ctx.responses_request_arc();
+            self.prepare_responses(ctx, &request_arc).await?;
+        } else {
+            error!(
+                function = "HarmonyPreparationStage::execute",
+                "Unsupported request type for Harmony pipeline"
+            );
+            return Err(error::bad_request(
+                "harmony_request_type_invalid",
+                "Only Chat and Responses requests supported in Harmony pipeline".to_string(),
+            ));
+        }
+
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "HarmonyPreparation"
+    }
+}
+
+impl HarmonyPreparationStage {
+    /// Prepare a chat completion request using Harmony encoding
+    async fn prepare_chat(
+        &self,
+        ctx: &mut RequestContext,
+        request: &ChatCompletionRequest,
+    ) -> Result<Option<Response>, Response> {
+        // Validate - reject logprobs
+        if request.logprobs {
+            error!(
+                function = "prepare_chat",
+                "logprobs requested but not supported for Harmony models"
+            );
+            return Err(error::bad_request(
+                "harmony_logprobs_not_supported",
+                "logprobs are not supported for Harmony models".to_string(),
+            ));
+        }
+
+        // Step 1: Filter tools if needed
+        let body_ref = utils::filter_chat_request_by_tool_choice(request);
+
+        // Step 2: Build tool constraints
+        let tool_constraints = if let Some(tools) = body_ref.tools.as_ref() {
+            Self::generate_tool_call_constraint(tools, &body_ref.tool_choice).map_err(|e| *e)?
+        } else {
+            None
+        };
+
+        // Step 3: Build via Harmony
+        let build_output = self.builder.build_from_chat(&body_ref).map_err(|e| {
+            error!(
+                function = "prepare_chat",
+                error = %e,
+                "Harmony build failed for chat request"
+            );
+            error::bad_request(
+                "harmony_build_failed",
+                format!("Harmony build failed: {}", e),
+            )
+        })?;
+
+        // Step 4: Store results
+        ctx.state.preparation = Some(PreparationOutput {
+            original_text: None,
+            token_ids: build_output.input_ids,
+            processed_messages: None,
+            tool_constraints,
+            filtered_request: if matches!(body_ref, std::borrow::Cow::Owned(_)) {
+                Some(body_ref.into_owned())
+            } else {
+                None
+            },
+            harmony_mode: true,
+            selection_text: Some(build_output.selection_text),
+            harmony_messages: Some(build_output.harmony_messages),
+            harmony_stop_ids: Some(build_output.stop_token_ids),
+        });
+
+        Ok(None)
+    }
+
+    /// Prepare a responses API request using Harmony encoding
+    ///
+    /// For responses API, we build from conversation history using the same Harmony
+    /// encoding that the builder provides. This handles the MCP loop integration.
+    pub async fn prepare_responses(
+        &self,
+        ctx: &mut RequestContext,
+        request: &ResponsesRequest,
+    ) -> Result<Option<Response>, Response> {
+        // Step 1: Extract function and MCP tools with schemas from ResponseTools
+        let mut function_tools = extract_tools_from_response_tools(request.tools.as_deref(), true);
+
+        // Step 2: Filter tools based on tool_choice (AllowedTools or Function)
+        // Note: Tool existence is already validated in ResponsesRequest::validate()
+        if let Some(filtered) =
+            utils::filter_tools_by_tool_choice(&function_tools, &request.tool_choice)
+        {
+            function_tools = filtered;
+        }
+
+        // Step 3: Generate Harmony structural tags
+        let tool_constraint = if !function_tools.is_empty() {
+            Self::generate_tool_call_constraint(&function_tools, &request.tool_choice)
+                .map_err(|e| *e)?
+        } else {
+            None
+        };
+
+        let text_constraint = if let Some(text_config) = &request.text {
+            Self::generate_text_format_constraint(text_config).map_err(|e| *e)?
+        } else {
+            None
+        };
+
+        if tool_constraint.is_some() && text_constraint.is_some() {
+            error!(
+                function = "prepare_responses",
+                "Conflicting constraints: both tool_choice and text format specified"
+            );
+            return Err(error::bad_request(
+                "conflicting_constraints",
+                "Cannot use both tool_choice (required/function) and text format (json_object/json_schema) simultaneously".to_string(),
+            ));
+        }
+
+        let constraint = tool_constraint.or(text_constraint);
+
+        // Step 3: Build via Harmony from responses API request
+        let build_output = self.builder.build_from_responses(request).map_err(|e| {
+            error!(
+                function = "prepare_responses",
+                error = %e,
+                "Harmony build failed for responses request"
+            );
+            error::bad_request(
+                "harmony_build_failed",
+                format!("Harmony build failed: {}", e),
+            )
+        })?;
+
+        // Step 4: Store results with constraint
+        ctx.state.preparation = Some(PreparationOutput {
+            original_text: None,
+            token_ids: build_output.input_ids,
+            processed_messages: None,
+            tool_constraints: constraint,
+            filtered_request: None,
+            harmony_mode: true,
+            selection_text: Some(build_output.selection_text),
+            harmony_messages: Some(build_output.harmony_messages),
+            harmony_stop_ids: Some(build_output.stop_token_ids),
+        });
+
+        Ok(None)
+    }
+
+    /// Generate Harmony structural tag for structured output (text field)
+    ///
+    /// Converts text.format to structural tag that constrains the final channel.
+    /// Returns None if text.format is not specified or is "text".
+    fn generate_text_format_constraint(
+        text_config: &crate::protocols::responses::TextConfig,
+    ) -> Result<Option<(String, String)>, Box<Response>> {
+        use crate::protocols::responses::TextFormat;
+
+        let Some(format) = &text_config.format else {
+            return Ok(None);
+        };
+
+        match format {
+            TextFormat::Text => Ok(None),
+            TextFormat::JsonObject => {
+                let tag = build_text_format_structural_tag(&serde_json::json!({"type": "object"}))
+                    .map_err(|e| {
+                        error!(
+                            function = "generate_text_format_constraint",
+                            error = %e,
+                            "Failed to build text format structural tag for JsonObject"
+                        );
+                        Box::new(error::internal_error("build_text_format_tag_failed", e))
+                    })?;
+                Ok(Some(("structural_tag".to_string(), tag)))
+            }
+            TextFormat::JsonSchema { schema, .. } => {
+                let tag = build_text_format_structural_tag(schema).map_err(|e| {
+                    error!(
+                        function = "generate_text_format_constraint",
+                        error = %e,
+                        "Failed to build text format structural tag for JsonSchema"
+                    );
+                    Box::new(error::internal_error("build_text_format_tag_failed", e))
+                })?;
+                Ok(Some(("structural_tag".to_string(), tag)))
+            }
+        }
+    }
+
+    /// Generate Harmony structural tag for tool constraints
+    ///
+    /// Uses structural tags with `triggered_tags` format to force Harmony format output.
+    /// This ensures the model outputs in Harmony format (with channels) even when constrained.
+    fn generate_tool_call_constraint(
+        tools: &[Tool],
+        tool_choice: &Option<ToolChoice>,
+    ) -> Result<Option<(String, String)>, Box<Response>> {
+        let Some(choice) = tool_choice.as_ref() else {
+            return Ok(None);
+        };
+
+        match choice {
+            ToolChoice::Function { function, .. } => {
+                let tag = Self::build_tool_call_structural_tag(tools, Some(&function.name))?;
+                Ok(Some(("structural_tag".to_string(), tag)))
+            }
+            ToolChoice::Value(ToolChoiceValue::Required) => {
+                let tag = Self::build_tool_call_structural_tag(tools, None)?;
+                Ok(Some(("structural_tag".to_string(), tag)))
+            }
+            ToolChoice::AllowedTools { mode, .. } => {
+                if mode == "required" {
+                    let tag = Self::build_tool_call_structural_tag(tools, None)?;
+                    Ok(Some(("structural_tag".to_string(), tag)))
+                } else {
+                    Ok(None)
+                }
+            }
+            _ => Ok(None),
+        }
+    }
+
+    /// Build Harmony structural tag for tool calling constraints
+    ///
+    /// Supports both reasoning-enabled and reasoning-disabled modes:
+    /// - With reasoning: triggers on `<|start|>assistant<|channel|>commentary` (waits for analysis)
+    /// - Without reasoning: triggers on `<|channel|>commentary` (goes directly to commentary)
+    fn build_tool_call_structural_tag(
+        tools: &[Tool],
+        specific_function: Option<&str>,
+    ) -> Result<String, Box<Response>> {
+        let mut tags = Vec::new();
+
+        // Filter tools if specific function requested
+        let tools_to_use: Vec<&Tool> = if let Some(func_name) = specific_function {
+            tools
+                .iter()
+                .filter(|t| t.function.name == func_name)
+                .collect()
+        } else {
+            tools.iter().collect()
+        };
+
+        // Validate specific function exists
+        match specific_function {
+            Some(tool_name) if tools_to_use.is_empty() => {
+                error!(
+                    function = "generate_tool_call_constraint",
+                    tool_name = %tool_name,
+                    "Specified tool not found in tools list"
+                );
+                return Err(Box::new(error::bad_request(
+                    "tool_not_found",
+                    format!("Tool '{}' not found in tools list", tool_name),
+                )));
+            }
+            _ => {}
+        }
+
+        // Build tags for each tool - need two patterns per tool for reasoning on/off
+        for tool in tools_to_use {
+            let tool_name = &tool.function.name;
+            let params_schema = &tool.function.parameters;
+
+            // Pattern 1: For reasoning-enabled mode (with analysis channel before commentary)
+            tags.push(json!({
+                "begin": format!("<|start|>assistant<|channel|>commentary to=functions.{}<|constrain|>json<|message|>", tool_name),
+                "content": {
+                    "type": "json_schema",
+                    "json_schema": params_schema
+                },
+                "end": "" // `end` is empty because <|call|> comes naturally from Harmony stop tokens
+            }));
+
+            // Pattern 2: For reasoning-disabled mode (goes directly to commentary channel)
+            tags.push(json!({
+                "begin": format!("<|channel|>commentary to=functions.{}<|constrain|>json<|message|>", tool_name),
+                "content": {
+                    "type": "json_schema",
+                    "json_schema": params_schema
+                },
+                "end": ""
+            }));
+        }
+
+        let stop_after_first = specific_function.is_some();
+
+        let structural_tag = json!({
+            "format": {
+                "type": "triggered_tags",
+                "triggers": ["<|start|>assistant<|channel|>commentary", "<|channel|>commentary"],
+                "tags": tags,
+                "at_least_one": true,
+                "stop_after_first": stop_after_first
+            }
+        });
+
+        serde_json::to_string(&structural_tag).map_err(|e| {
+            error!(
+                function = "generate_tool_call_constraint",
+                error = %e,
+                "Failed to serialize structural tag"
+            );
+            Box::new(error::internal_error(
+                "serialize_structural_tag_failed",
+                format!("Failed to serialize structural tag: {}", e),
+            ))
+        })
+    }
+}
+
+/// Build Harmony structural tag for structured output (JSON schema constraint)
+///
+/// Creates a structural tag that applies JSON schema constraint to the final channel,
+/// supporting both reasoning-enabled and reasoning-disabled modes:
+/// - With reasoning: triggers on `<|start|>assistant<|channel|>final` (waits for analysis to complete)
+/// - Without reasoning: triggers on `<|channel|>final` (goes directly to final channel)
+///
+/// This is used for the Responses API text.format field (json_object or json_schema).
+pub(crate) fn build_text_format_structural_tag(
+    schema: &serde_json::Value,
+) -> Result<String, String> {
+    let structural_tag = json!({
+        "format": {
+            "type": "triggered_tags",
+            "triggers": ["<|start|>assistant<|channel|>final", "<|channel|>final"],
+            "tags": [
+                {
+                    // Pattern 1: For reasoning-enabled mode (with analysis channel before final)
+                    "begin": "<|start|>assistant<|channel|>final<|constrain|>json<|message|>",
+                    "content": {
+                        "type": "json_schema",
+                        "json_schema": schema
+                    },
+                    "end": ""
+                },
+                {
+                    // Pattern 2: For reasoning-disabled mode (goes directly to final channel)
+                    "begin": "<|channel|>final<|constrain|>json<|message|>",
+                    "content": {
+                        "type": "json_schema",
+                        "json_schema": schema
+                    },
+                    "end": ""
+                }
+            ],
+            "at_least_one": true,
+            "stop_after_first": true
+        }
+    });
+
+    serde_json::to_string(&structural_tag).map_err(|e| {
+        format!(
+            "Failed to serialize structural tag for structured output: {}",
+            e
+        )
+    })
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/stages/request_building.rs

@@ -0,0 +1,199 @@
+//! Harmony Request Building Stage: Build gRPC request from Harmony-encoded tokens
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::{debug, error};
+use uuid::Uuid;
+
+use crate::routers::{
+    error,
+    grpc::{
+        common::stages::{helpers, PipelineStage},
+        context::{ClientSelection, RequestContext, RequestType, WorkerSelection},
+        proto_wrapper::{ProtoGenerateRequest, ProtoRequest},
+    },
+};
+
+/// Harmony Request Building stage: Convert Harmony tokens to gRPC request
+///
+/// Takes the Harmony-encoded input_ids from preparation and builds a proto::GenerateRequest.
+/// Unlike regular request building, this uses token_ids directly (Harmony encoding handles messages).
+pub(crate) struct HarmonyRequestBuildingStage {
+    inject_pd_metadata: bool,
+}
+
+impl HarmonyRequestBuildingStage {
+    /// Create a new Harmony request building stage
+    pub fn new(inject_pd_metadata: bool) -> Self {
+        Self { inject_pd_metadata }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for HarmonyRequestBuildingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        // Get preparation output
+        let prep = ctx.state.preparation.as_ref().ok_or_else(|| {
+            error!(
+                function = "HarmonyRequestBuildingStage::execute",
+                "Preparation stage not completed"
+            );
+            error::internal_error("preparation_not_completed", "Preparation not completed")
+        })?;
+
+        // Get clients
+        let clients = ctx.state.clients.as_ref().ok_or_else(|| {
+            error!(
+                function = "HarmonyRequestBuildingStage::execute",
+                "Client acquisition stage not completed"
+            );
+            error::internal_error(
+                "client_acquisition_not_completed",
+                "Client acquisition not completed",
+            )
+        })?;
+        let builder_client = match clients {
+            ClientSelection::Single { client } => client,
+            ClientSelection::Dual { prefill, .. } => prefill,
+        };
+
+        // Harmony model support not yet implemented for vLLM
+        if builder_client.is_vllm() {
+            return Err(error::not_implemented(
+                "harmony_vllm_not_supported",
+                "Harmony model support is not yet implemented for vLLM backend. \
+                 Please use runtime_type: sglang for Harmony models.",
+            ));
+        }
+
+        // Generate request_id based on request type
+        let request_id = match &ctx.input.request_type {
+            RequestType::Chat(_) => format!("chatcmpl-{}", Uuid::new_v4()),
+            RequestType::Responses(_) => format!("responses-{}", Uuid::new_v4()),
+            RequestType::Generate(_) => {
+                error!(
+                    function = "HarmonyRequestBuildingStage::execute",
+                    "Generate request type not supported for Harmony models"
+                );
+                return Err(error::bad_request(
+                    "harmony_generate_not_supported",
+                    "Generate requests are not supported with Harmony models".to_string(),
+                ));
+            }
+            RequestType::Embedding(_) => {
+                error!(
+                    function = "HarmonyRequestBuildingStage::execute",
+                    "Embedding requests not supported for Harmony models"
+                );
+                return Err(error::bad_request(
+                    "harmony_embedding_not_supported",
+                    "Embedding requests are not supported with Harmony models".to_string(),
+                ));
+            }
+            RequestType::Classify(_) => {
+                error!(
+                    function = "HarmonyRequestBuildingStage::execute",
+                    "Classify requests not supported for Harmony models"
+                );
+                return Err(error::bad_request(
+                    "harmony_classify_not_supported",
+                    "Classify requests are not supported with Harmony models".to_string(),
+                ));
+            }
+        };
+
+        // Build gRPC request using token_ids directly (Harmony encoding already handled message rendering)
+        let placeholder_processed_text = "[harmony]".to_string();
+
+        // Harmony is SGLang-only, so we can safely unwrap as SGLang
+        let sglang_client = builder_client.as_sglang();
+        let proto_request_inner = match &ctx.input.request_type {
+            RequestType::Chat(request) => {
+                // Use filtered request if present from preparation; otherwise original
+                let body = prep.filtered_request.as_ref().unwrap_or(request.as_ref());
+
+                sglang_client
+                    .build_generate_request_from_chat(
+                        request_id,
+                        body,
+                        placeholder_processed_text,
+                        prep.token_ids.clone(),
+                        None,
+                        prep.tool_constraints.clone(),
+                    )
+                    .map_err(|e| {
+                        error!(
+                            function = "HarmonyRequestBuildingStage::execute",
+                            error = %e,
+                            "Failed to build generate request from chat"
+                        );
+                        error::bad_request(
+                            "invalid_request_parameters",
+                            format!("Invalid request parameters: {}", e),
+                        )
+                    })?
+            }
+            RequestType::Responses(request) => sglang_client
+                .build_generate_request_from_responses(
+                    request_id,
+                    request.as_ref(),
+                    placeholder_processed_text,
+                    prep.token_ids.clone(),
+                    prep.harmony_stop_ids.clone(),
+                    prep.tool_constraints.clone(),
+                )
+                .map_err(|e| {
+                    error!(
+                        function = "HarmonyRequestBuildingStage::execute",
+                        error = %e,
+                        "Failed to build generate request from responses"
+                    );
+                    error::bad_request(
+                        "invalid_request_parameters",
+                        format!("Invalid request parameters: {}", e),
+                    )
+                })?,
+            RequestType::Embedding(_) => {
+                error!(
+                    function = "HarmonyRequestBuildingStage::execute",
+                    "Embedding requests not supported for Harmony models"
+                );
+                return Err(error::bad_request(
+                    "harmony_embedding_not_supported",
+                    "Embedding requests are not supported with Harmony models".to_string(),
+                ));
+            }
+            _ => unreachable!(), // All other request types should be handled above
+        };
+
+        let mut proto_request = ProtoGenerateRequest::Sglang(Box::new(proto_request_inner));
+
+        // Inject Harmony stop token IDs into sampling params for ALL Harmony requests
+        // These stop tokens (<|return|> and <|call|>) prevent the model from generating
+        // malformed Harmony sequences
+        if let Some(harmony_stops) = &prep.harmony_stop_ids {
+            let sglang_req = proto_request.as_sglang_mut();
+            if let Some(params) = sglang_req.sampling_params.as_mut() {
+                params.stop_token_ids.extend_from_slice(harmony_stops);
+                debug!(
+                    stop_token_count = harmony_stops.len(),
+                    "Injected Harmony stop tokens into sampling params"
+                );
+            }
+        }
+
+        // Inject PD metadata if needed
+        if self.inject_pd_metadata {
+            if let Some(WorkerSelection::Dual { prefill, .. }) = ctx.state.workers.as_ref() {
+                helpers::inject_bootstrap_metadata(&mut proto_request, prefill);
+            }
+        }
+
+        ctx.state.proto_request = Some(ProtoRequest::Generate(proto_request));
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "HarmonyRequestBuilding"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/stages/response_processing.rs

@@ -0,0 +1,168 @@
+//! Harmony Response Processing Stage: Parse Harmony channels to ChatCompletionResponse
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use super::super::{HarmonyResponseProcessor, HarmonyStreamingProcessor};
+use crate::{
+    core::AttachedBody,
+    routers::{
+        error,
+        grpc::{
+            common::stages::PipelineStage,
+            context::{FinalResponse, RequestContext, RequestType},
+        },
+    },
+};
+
+/// Harmony Response Processing stage: Parse and format Harmony responses
+///
+/// Takes output tokens from execution and parses them using HarmonyParserAdapter
+/// to extract analysis, tool calls, and final response text from Harmony channels.
+pub(crate) struct HarmonyResponseProcessingStage {
+    processor: HarmonyResponseProcessor,
+    streaming_processor: Arc<HarmonyStreamingProcessor>,
+}
+
+impl HarmonyResponseProcessingStage {
+    /// Create a new Harmony response processing stage
+    pub fn new() -> Self {
+        Self {
+            processor: HarmonyResponseProcessor::new(),
+            streaming_processor: Arc::new(HarmonyStreamingProcessor::new()),
+        }
+    }
+}
+
+impl Default for HarmonyResponseProcessingStage {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl PipelineStage for HarmonyResponseProcessingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let is_streaming = ctx.is_streaming();
+
+        // Check request type to determine which processor method to call
+        match &ctx.input.request_type {
+            RequestType::Chat(_) => {
+                // Get execution result (output tokens from model)
+                let execution_result =
+                    ctx.state.response.execution_result.take().ok_or_else(|| {
+                        error!(
+                            function = "HarmonyResponseProcessingStage::execute",
+                            request_type = "Chat",
+                            "No execution result available"
+                        );
+                        error::internal_error("no_execution_result", "No execution result")
+                    })?;
+
+                let dispatch = ctx.state.dispatch.as_ref().cloned().ok_or_else(|| {
+                    error!(
+                        function = "HarmonyResponseProcessingStage::execute",
+                        request_type = "Chat",
+                        "Dispatch metadata not set"
+                    );
+                    error::internal_error("dispatch_metadata_not_set", "Dispatch metadata not set")
+                })?;
+
+                // For streaming, delegate to streaming processor and return SSE response
+                if is_streaming {
+                    let response = self
+                        .streaming_processor
+                        .clone()
+                        .process_streaming_chat_response(
+                            execution_result,
+                            ctx.chat_request_arc(),
+                            dispatch,
+                        );
+
+                    // Attach load guards to response body for proper RAII lifecycle
+                    let response = match ctx.state.load_guards.take() {
+                        Some(guards) => AttachedBody::wrap_response(response, guards),
+                        None => response,
+                    };
+
+                    return Ok(Some(response));
+                }
+
+                // For non-streaming, delegate to Harmony response processor to build ChatCompletionResponse
+                let chat_request = ctx.chat_request_arc();
+                let response = self
+                    .processor
+                    .process_non_streaming_chat_response(execution_result, chat_request, dispatch)
+                    .await?;
+
+                ctx.state.response.final_response = Some(FinalResponse::Chat(response));
+                Ok(None)
+            }
+            RequestType::Responses(_) => {
+                // For streaming Responses API, leave execution_result in context
+                // for external streaming processor (serve_harmony_responses_stream)
+                if is_streaming {
+                    // Don't take execution_result - let the caller handle it
+                    return Ok(None);
+                }
+
+                // For non-streaming, process normally
+                let execution_result =
+                    ctx.state.response.execution_result.take().ok_or_else(|| {
+                        error!(
+                            function = "HarmonyResponseProcessingStage::execute",
+                            request_type = "Responses",
+                            "No execution result available"
+                        );
+                        error::internal_error("no_execution_result", "No execution result")
+                    })?;
+
+                let dispatch = ctx.state.dispatch.as_ref().cloned().ok_or_else(|| {
+                    error!(
+                        function = "HarmonyResponseProcessingStage::execute",
+                        request_type = "Responses",
+                        "Dispatch metadata not set"
+                    );
+                    error::internal_error("dispatch_metadata_not_set", "Dispatch metadata not set")
+                })?;
+
+                let responses_request = ctx.responses_request_arc();
+                let iteration_result = self
+                    .processor
+                    .process_responses_iteration(execution_result, responses_request, dispatch)
+                    .await?;
+
+                ctx.state.response.responses_iteration_result = Some(iteration_result);
+                Ok(None)
+            }
+            RequestType::Generate(_) | RequestType::Embedding(_) | RequestType::Classify(_) => {
+                error!(
+                    function = "HarmonyResponseProcessingStage::execute",
+                    "Generate/Embedding/Classify request type not supported in Harmony pipeline"
+                );
+                Err(error::internal_error(
+                    "requests_not_supported_in_harmony",
+                    "Generate/Embedding/Classify requests not supported in Harmony pipeline",
+                ))
+            }
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "HarmonyResponseProcessing"
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_response_processing_stage_creation() {
+        let stage = HarmonyResponseProcessingStage::new();
+        assert_eq!(stage.name(), "HarmonyResponseProcessing");
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/harmony/types.rs

@@ -0,0 +1,142 @@
+//! Shared types for Harmony pipeline
+
+use openai_harmony::chat::Content;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use crate::protocols::common::ToolCall;
+
+/// Harmony message format
+///
+/// Represents messages in the Harmony encoding format with role and content.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct HarmonyMessage {
+    pub role: String,
+    pub content: String,
+}
+
+/// Some methods are kept for API completeness even if currently unused.
+#[allow(dead_code)]
+impl HarmonyMessage {
+    pub fn new(role: impl Into<String>, content: impl Into<String>) -> Self {
+        Self {
+            role: role.into(),
+            content: content.into(),
+        }
+    }
+
+    pub fn user(content: impl Into<String>) -> Self {
+        Self::new("user", content)
+    }
+
+    pub fn assistant(content: impl Into<String>) -> Self {
+        Self::new("assistant", content)
+    }
+
+    pub fn system(content: impl Into<String>) -> Self {
+        Self::new("system", content)
+    }
+
+    /// Convert from openai_harmony::chat::Message to our simplified HarmonyMessage
+    pub fn from_openai_harmony(msg: openai_harmony::chat::Message) -> Self {
+        // Extract role as string
+        let role = match msg.author.role {
+            openai_harmony::chat::Role::User => "user",
+            openai_harmony::chat::Role::Assistant => "assistant",
+            openai_harmony::chat::Role::System => "system",
+            openai_harmony::chat::Role::Developer => "developer",
+            openai_harmony::chat::Role::Tool => "tool",
+        }
+        .to_string();
+
+        // Extract text content from all Content::Text parts
+        let content = msg
+            .content
+            .iter()
+            .filter_map(|c| match c {
+                Content::Text(tc) => Some(tc.text.as_str()),
+                _ => None,
+            })
+            .collect::<Vec<_>>()
+            .join("");
+
+        Self { role, content }
+    }
+}
+
+/// Output from Harmony encoding process
+///
+/// Contains the encoded input_ids, stop tokens, selection text for worker routing,
+/// and the Harmony message history.
+#[derive(Debug, Clone)]
+pub(crate) struct HarmonyBuildOutput {
+    /// Encoded token IDs to send to the model
+    pub input_ids: Vec<u32>,
+
+    /// Stop token IDs for this model (injected into sampling params)
+    pub stop_token_ids: Vec<u32>,
+
+    /// Selection text for worker routing (concise snippet from last user message)
+    pub selection_text: String,
+
+    /// Harmony messages for this conversation (used for history tracking)
+    pub harmony_messages: Vec<HarmonyMessage>,
+}
+
+/// Parsed output from all three Harmony channels
+///
+/// Represents the complete response after parsing analysis, commentary, and final channels.
+#[derive(Debug, Clone)]
+pub(crate) struct HarmonyChannelOutput {
+    /// Analysis/reasoning content (from analysis channel)
+    pub analysis: Option<String>,
+
+    /// Tool calls (from commentary channel)
+    pub commentary: Option<Vec<ToolCall>>,
+
+    /// Final text content (from final channel)
+    pub final_text: String,
+
+    /// Finish reason
+    pub finish_reason: String,
+
+    /// Matched stop token (if any)
+    pub matched_stop: Option<Value>,
+
+    /// Number of reasoning tokens (from analysis and commentary channels)
+    pub reasoning_token_count: u32,
+}
+
+/// Streaming delta for SSE responses
+///
+/// Represents incremental updates as tokens are parsed from the stream.
+#[derive(Debug, Clone)]
+pub(crate) struct HarmonyChannelDelta {
+    /// Delta for analysis/reasoning content
+    pub analysis_delta: Option<String>,
+
+    /// Delta for tool calls
+    pub commentary_delta: Option<ToolCallDelta>,
+
+    /// Delta for final text content
+    pub final_delta: Option<String>,
+
+    /// Whether this is the final delta
+    #[allow(dead_code)]
+    pub is_final: bool,
+}
+
+/// Tool call delta for streaming
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct ToolCallDelta {
+    pub index: usize,
+    pub id: Option<String>,
+    pub function: Option<FunctionDelta>,
+}
+
+/// Function call delta for streaming
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct FunctionDelta {
+    pub name: Option<String>,
+    pub arguments: Option<String>,
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/mod.rs

@@ -0,0 +1,25 @@
+//! gRPC router implementations
+
+use smg_grpc_client::sglang_proto::MultimodalInputs;
+
+use crate::protocols::common::StringOrArray;
+
+pub mod client; // Used by core/
+pub(crate) mod common;
+pub(crate) mod context;
+pub(crate) mod harmony;
+pub(crate) mod pd_router; // Used by routers/factory
+pub(crate) mod pipeline;
+pub(crate) mod proto_wrapper;
+pub(crate) mod regular;
+pub(crate) mod router; // Used by routers/factory
+pub(crate) mod utils; // Used by routers/http
+
+/// Processed chat messages ready for gRPC generation
+#[derive(Debug)]
+pub(crate) struct ProcessedMessages {
+    pub text: String,
+    pub multimodal_inputs: Option<MultimodalInputs>,
+    #[allow(dead_code)]
+    pub stop_sequences: Option<StringOrArray>,
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/pd_router.rs

@@ -0,0 +1,244 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use axum::{http::HeaderMap, response::Response};
+use tracing::debug;
+
+use super::{context::SharedComponents, pipeline::RequestPipeline};
+use crate::{
+    app_context::AppContext,
+    config::types::RetryConfig,
+    core::{
+        is_retryable_status, ConnectionMode, RetryExecutor, WorkerRegistry, WorkerType,
+        UNKNOWN_MODEL_ID,
+    },
+    observability::metrics::{metrics_labels, Metrics},
+    protocols::{chat::ChatCompletionRequest, generate::GenerateRequest},
+    routers::RouterTrait,
+};
+
+/// gRPC PD (Prefill-Decode) router implementation for SGLang
+#[derive(Clone)]
+pub struct GrpcPDRouter {
+    worker_registry: Arc<WorkerRegistry>,
+    pipeline: RequestPipeline,
+    shared_components: Arc<SharedComponents>,
+    retry_config: RetryConfig,
+}
+
+impl GrpcPDRouter {
+    /// Create a new gRPC PD router
+    pub async fn new(ctx: &Arc<AppContext>) -> Result<Self, String> {
+        // Get registries from context
+        let worker_registry = ctx.worker_registry.clone();
+        let policy_registry = ctx.policy_registry.clone();
+
+        // Get tokenizer registry (no longer requires pre-loaded tokenizer)
+        let tokenizer_registry = ctx.tokenizer_registry.clone();
+
+        let reasoning_parser_factory = ctx
+            .reasoning_parser_factory
+            .as_ref()
+            .ok_or_else(|| "gRPC PD router requires reasoning parser factory".to_string())?
+            .clone();
+        let tool_parser_factory = ctx
+            .tool_parser_factory
+            .as_ref()
+            .ok_or_else(|| "gRPC PD router requires tool parser factory".to_string())?
+            .clone();
+
+        // Create shared components for pipeline
+        let shared_components = Arc::new(SharedComponents {
+            tokenizer_registry: tokenizer_registry.clone(),
+            tool_parser_factory: tool_parser_factory.clone(),
+            reasoning_parser_factory: reasoning_parser_factory.clone(),
+        });
+
+        // Create PD pipeline
+        let pipeline = RequestPipeline::new_pd(
+            worker_registry.clone(),
+            policy_registry.clone(),
+            tool_parser_factory.clone(),
+            reasoning_parser_factory.clone(),
+            ctx.configured_tool_parser.clone(),
+            ctx.configured_reasoning_parser.clone(),
+        );
+
+        Ok(GrpcPDRouter {
+            worker_registry,
+            pipeline,
+            shared_components,
+            retry_config: ctx.router_config.effective_retry_config(),
+        })
+    }
+
+    /// Main route_generate implementation with PD dual dispatch
+    async fn route_generate_impl(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &GenerateRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        debug!(
+            "Processing generate request for model: {} (PD mode)",
+            model_id.unwrap_or(UNKNOWN_MODEL_ID)
+        );
+
+        // Clone values needed for retry closure
+        let request = Arc::new(body.clone());
+        let headers_cloned = headers.cloned();
+        let model_id_cloned = model_id.map(|s| s.to_string());
+        let components = self.shared_components.clone();
+        let pipeline = &self.pipeline;
+
+        RetryExecutor::execute_response_with_retry(
+            &self.retry_config,
+            |_attempt| {
+                let request = Arc::clone(&request);
+                let headers = headers_cloned.clone();
+                let model_id = model_id_cloned.clone();
+                let components = Arc::clone(&components);
+                async move {
+                    pipeline
+                        .execute_generate(request, headers, model_id, components)
+                        .await
+                }
+            },
+            |res, _attempt| is_retryable_status(res.status()),
+            |delay, attempt| {
+                Metrics::record_worker_retry(
+                    metrics_labels::WORKER_PREFILL,
+                    metrics_labels::ENDPOINT_GENERATE,
+                );
+                Metrics::record_worker_retry(
+                    metrics_labels::WORKER_DECODE,
+                    metrics_labels::ENDPOINT_GENERATE,
+                );
+                Metrics::record_worker_retry_backoff(attempt, delay);
+            },
+            || {
+                Metrics::record_worker_retries_exhausted(
+                    metrics_labels::WORKER_PREFILL,
+                    metrics_labels::ENDPOINT_GENERATE,
+                );
+                Metrics::record_worker_retries_exhausted(
+                    metrics_labels::WORKER_DECODE,
+                    metrics_labels::ENDPOINT_GENERATE,
+                );
+            },
+        )
+        .await
+    }
+
+    /// Main route_chat implementation with PD dual dispatch
+    async fn route_chat_impl(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ChatCompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        debug!(
+            "Processing chat completion request for model: {} (PD mode)",
+            model_id.unwrap_or(UNKNOWN_MODEL_ID)
+        );
+
+        // Clone values needed for retry closure
+        let request = Arc::new(body.clone());
+        let headers_cloned = headers.cloned();
+        let model_id_cloned = model_id.map(|s| s.to_string());
+        let components = self.shared_components.clone();
+        let pipeline = &self.pipeline;
+
+        RetryExecutor::execute_response_with_retry(
+            &self.retry_config,
+            |_attempt| {
+                let request = Arc::clone(&request);
+                let headers = headers_cloned.clone();
+                let model_id = model_id_cloned.clone();
+                let components = Arc::clone(&components);
+                async move {
+                    pipeline
+                        .execute_chat(request, headers, model_id, components)
+                        .await
+                }
+            },
+            |res, _attempt| is_retryable_status(res.status()),
+            |delay, attempt| {
+                Metrics::record_worker_retry(
+                    metrics_labels::WORKER_PREFILL,
+                    metrics_labels::ENDPOINT_CHAT,
+                );
+                Metrics::record_worker_retry(
+                    metrics_labels::WORKER_DECODE,
+                    metrics_labels::ENDPOINT_CHAT,
+                );
+                Metrics::record_worker_retry_backoff(attempt, delay);
+            },
+            || {
+                Metrics::record_worker_retries_exhausted(
+                    metrics_labels::WORKER_PREFILL,
+                    metrics_labels::ENDPOINT_CHAT,
+                );
+                Metrics::record_worker_retries_exhausted(
+                    metrics_labels::WORKER_DECODE,
+                    metrics_labels::ENDPOINT_CHAT,
+                );
+            },
+        )
+        .await
+    }
+}
+
+impl std::fmt::Debug for GrpcPDRouter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let prefill_workers = self.worker_registry.get_workers_filtered(
+            None,
+            Some(WorkerType::Prefill {
+                bootstrap_port: None,
+            }),
+            Some(ConnectionMode::Grpc { port: None }),
+            None,
+            false,
+        );
+        let decode_workers = self.worker_registry.get_workers_filtered(
+            None,
+            Some(WorkerType::Decode),
+            Some(ConnectionMode::Grpc { port: None }),
+            None,
+            false,
+        );
+        f.debug_struct("GrpcPDRouter")
+            .field("prefill_workers_count", &prefill_workers.len())
+            .field("decode_workers_count", &decode_workers.len())
+            .finish()
+    }
+}
+
+#[async_trait]
+impl RouterTrait for GrpcPDRouter {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    async fn route_generate(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &GenerateRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_generate_impl(headers, body, model_id).await
+    }
+
+    async fn route_chat(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ChatCompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_chat_impl(headers, body, model_id).await
+    }
+
+    fn router_type(&self) -> &'static str {
+        "grpc_pd"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/pipeline.rs

@@ -0,0 +1,868 @@
+//! Pipeline orchestrator for gRPC router request processing
+//!
+//! This module defines the RequestPipeline orchestrator that coordinates
+//! the execution of pipeline stages from request preparation to response delivery.
+
+use std::{sync::Arc, time::Instant};
+
+use axum::response::{IntoResponse, Response};
+use tracing::{debug, error};
+
+// Import embedding-specific and classify-specific stages
+use super::regular::stages::classify::ClassifyResponseProcessingStage;
+use super::{
+    common::{responses::ResponsesContext, stages::*},
+    context::*,
+    harmony,
+    regular::{
+        processor,
+        stages::{
+            embedding::{
+                preparation::EmbeddingPreparationStage,
+                request_building::EmbeddingRequestBuildingStage,
+                response_processing::EmbeddingResponseProcessingStage,
+            },
+            *,
+        },
+        streaming,
+    },
+    utils::error_type_from_status,
+};
+use crate::{
+    core::{WorkerRegistry, UNKNOWN_MODEL_ID},
+    observability::metrics::{bool_to_static_str, metrics_labels, Metrics},
+    policies::PolicyRegistry,
+    protocols::{
+        chat::{ChatCompletionRequest, ChatCompletionResponse},
+        classify::ClassifyRequest,
+        embedding::EmbeddingRequest,
+        generate::GenerateRequest,
+    },
+    reasoning_parser::ParserFactory as ReasoningParserFactory,
+    routers::error,
+    tool_parser::ParserFactory as ToolParserFactory,
+};
+
+/// Generic request pipeline for all request types
+///
+/// Orchestrates all stages from request preparation to response delivery.
+/// Configured differently for regular vs PD mode.
+#[derive(Clone)]
+pub(crate) struct RequestPipeline {
+    stages: Arc<Vec<Box<dyn PipelineStage>>>,
+    /// Backend type for metrics labeling
+    backend_type: &'static str,
+}
+
+impl RequestPipeline {
+    /// Create a regular (single-worker) pipeline
+    pub fn new_regular(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+        tool_parser_factory: ToolParserFactory,
+        reasoning_parser_factory: ReasoningParserFactory,
+        configured_tool_parser: Option<String>,
+        configured_reasoning_parser: Option<String>,
+    ) -> Self {
+        let processor = processor::ResponseProcessor::new(
+            tool_parser_factory.clone(),
+            reasoning_parser_factory.clone(),
+            configured_tool_parser.clone(),
+            configured_reasoning_parser.clone(),
+        );
+
+        let streaming_processor = Arc::new(streaming::StreamingProcessor::new(
+            tool_parser_factory,
+            reasoning_parser_factory,
+            configured_tool_parser,
+            configured_reasoning_parser,
+            metrics_labels::BACKEND_REGULAR,
+        ));
+
+        let stages: Vec<Box<dyn PipelineStage>> = vec![
+            Box::new(PreparationStage::new()),
+            Box::new(WorkerSelectionStage::new(
+                worker_registry,
+                policy_registry,
+                WorkerSelectionMode::Regular,
+            )),
+            Box::new(ClientAcquisitionStage),
+            Box::new(RequestBuildingStage::new(false)), // No PD metadata
+            Box::new(DispatchMetadataStage),
+            Box::new(RequestExecutionStage::new(ExecutionMode::Single)),
+            Box::new(ResponseProcessingStage::new(processor, streaming_processor)),
+        ];
+
+        Self {
+            stages: Arc::new(stages),
+            backend_type: metrics_labels::BACKEND_REGULAR,
+        }
+    }
+
+    /// Create a Harmony (single-worker) pipeline for Harmony-capable models
+    pub fn new_harmony(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+        _tool_parser_factory: ToolParserFactory,
+        _reasoning_parser_factory: ReasoningParserFactory,
+        _configured_tool_parser: Option<String>,
+        _configured_reasoning_parser: Option<String>,
+    ) -> Self {
+        let stages: Vec<Box<dyn PipelineStage>> = vec![
+            Box::new(harmony::stages::HarmonyPreparationStage::new()),
+            Box::new(WorkerSelectionStage::new(
+                worker_registry,
+                policy_registry,
+                WorkerSelectionMode::Regular,
+            )),
+            Box::new(ClientAcquisitionStage),
+            Box::new(harmony::stages::HarmonyRequestBuildingStage::new(false)),
+            Box::new(DispatchMetadataStage),
+            Box::new(RequestExecutionStage::new(ExecutionMode::Single)),
+            Box::new(harmony::stages::HarmonyResponseProcessingStage::new()),
+        ];
+
+        Self {
+            stages: Arc::new(stages),
+            backend_type: metrics_labels::BACKEND_REGULAR,
+        }
+    }
+
+    /// Create a Harmony PD (prefill-decode) pipeline
+    #[allow(dead_code)]
+    pub fn new_harmony_pd(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+        _tool_parser_factory: ToolParserFactory,
+        _reasoning_parser_factory: ReasoningParserFactory,
+        _configured_tool_parser: Option<String>,
+        _configured_reasoning_parser: Option<String>,
+    ) -> Self {
+        let stages: Vec<Box<dyn PipelineStage>> = vec![
+            Box::new(harmony::stages::HarmonyPreparationStage::new()),
+            Box::new(WorkerSelectionStage::new(
+                worker_registry,
+                policy_registry,
+                WorkerSelectionMode::PrefillDecode,
+            )),
+            Box::new(ClientAcquisitionStage),
+            Box::new(harmony::stages::HarmonyRequestBuildingStage::new(true)),
+            Box::new(DispatchMetadataStage),
+            Box::new(RequestExecutionStage::new(ExecutionMode::DualDispatch)),
+            Box::new(harmony::stages::HarmonyResponseProcessingStage::new()),
+        ];
+
+        Self {
+            stages: Arc::new(stages),
+            backend_type: metrics_labels::BACKEND_PD,
+        }
+    }
+
+    /// Create a PD (prefill-decode) pipeline
+    pub fn new_pd(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+        tool_parser_factory: ToolParserFactory,
+        reasoning_parser_factory: ReasoningParserFactory,
+        configured_tool_parser: Option<String>,
+        configured_reasoning_parser: Option<String>,
+    ) -> Self {
+        let processor = processor::ResponseProcessor::new(
+            tool_parser_factory.clone(),
+            reasoning_parser_factory.clone(),
+            configured_tool_parser.clone(),
+            configured_reasoning_parser.clone(),
+        );
+
+        let streaming_processor = Arc::new(streaming::StreamingProcessor::new(
+            tool_parser_factory,
+            reasoning_parser_factory,
+            configured_tool_parser,
+            configured_reasoning_parser,
+            metrics_labels::BACKEND_PD,
+        ));
+
+        let stages: Vec<Box<dyn PipelineStage>> = vec![
+            Box::new(PreparationStage::new()),
+            Box::new(WorkerSelectionStage::new(
+                worker_registry,
+                policy_registry,
+                WorkerSelectionMode::PrefillDecode,
+            )),
+            Box::new(ClientAcquisitionStage),
+            Box::new(RequestBuildingStage::new(true)), // Inject PD metadata
+            Box::new(DispatchMetadataStage),
+            Box::new(RequestExecutionStage::new(ExecutionMode::DualDispatch)),
+            Box::new(ResponseProcessingStage::new(processor, streaming_processor)),
+        ];
+
+        Self {
+            stages: Arc::new(stages),
+            backend_type: metrics_labels::BACKEND_PD,
+        }
+    }
+
+    /// Create an embeddings pipeline
+    pub fn new_embeddings(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+    ) -> Self {
+        let stages: Vec<Box<dyn PipelineStage>> = vec![
+            Box::new(EmbeddingPreparationStage::new()),
+            Box::new(WorkerSelectionStage::new(
+                worker_registry,
+                policy_registry,
+                WorkerSelectionMode::Regular, // Embeddings are always single
+            )),
+            Box::new(ClientAcquisitionStage),
+            Box::new(EmbeddingRequestBuildingStage::new()),
+            Box::new(DispatchMetadataStage),
+            Box::new(RequestExecutionStage::new(ExecutionMode::Single)),
+            Box::new(EmbeddingResponseProcessingStage::new()),
+        ];
+
+        Self {
+            stages: Arc::new(stages),
+            backend_type: metrics_labels::BACKEND_REGULAR, // Embeddings are regular for now
+        }
+    }
+
+    /// Create a classify pipeline
+    ///
+    /// Classify reuses embedding stages for preparation and request building,
+    /// but uses its own response processing for softmax + label mapping.
+    pub fn new_classify(
+        worker_registry: Arc<WorkerRegistry>,
+        policy_registry: Arc<PolicyRegistry>,
+    ) -> Self {
+        let stages: Vec<Box<dyn PipelineStage>> = vec![
+            Box::new(EmbeddingPreparationStage::new()),
+            Box::new(WorkerSelectionStage::new(
+                worker_registry,
+                policy_registry,
+                WorkerSelectionMode::Regular, // Classify is always single worker
+            )),
+            Box::new(ClientAcquisitionStage),
+            Box::new(EmbeddingRequestBuildingStage::new()),
+            Box::new(DispatchMetadataStage),
+            Box::new(RequestExecutionStage::new(ExecutionMode::Single)),
+            Box::new(ClassifyResponseProcessingStage::new()),
+        ];
+
+        Self {
+            stages: Arc::new(stages),
+            backend_type: metrics_labels::BACKEND_REGULAR,
+        }
+    }
+
+    /// Execute the complete pipeline for a chat request
+    pub async fn execute_chat(
+        &self,
+        request: Arc<ChatCompletionRequest>,
+        headers: Option<http::HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Response {
+        let start = Instant::now();
+        // Clone Arc for metrics (cheap atomic increment) to avoid borrow issues
+        let request_for_metrics = Arc::clone(&request);
+        let streaming = request.stream;
+
+        // Record request start
+        Metrics::record_router_request(
+            metrics_labels::ROUTER_GRPC,
+            self.backend_type,
+            metrics_labels::CONNECTION_GRPC,
+            &request_for_metrics.model,
+            metrics_labels::ENDPOINT_CHAT,
+            bool_to_static_str(streaming),
+        );
+
+        let mut ctx = RequestContext::for_chat(request, headers, model_id, components);
+
+        for stage in self.stages.iter() {
+            match stage.execute(&mut ctx).await {
+                Ok(Some(response)) => {
+                    // Stage completed with streaming response - record success and return
+                    Metrics::record_router_duration(
+                        metrics_labels::ROUTER_GRPC,
+                        self.backend_type,
+                        metrics_labels::CONNECTION_GRPC,
+                        &request_for_metrics.model,
+                        metrics_labels::ENDPOINT_CHAT,
+                        start.elapsed(),
+                    );
+                    return response;
+                }
+                Ok(None) => continue,
+                Err(response) => {
+                    Metrics::record_router_error(
+                        metrics_labels::ROUTER_GRPC,
+                        self.backend_type,
+                        metrics_labels::CONNECTION_GRPC,
+                        &request_for_metrics.model,
+                        metrics_labels::ENDPOINT_CHAT,
+                        error_type_from_status(response.status()),
+                    );
+                    error!(
+                        "Stage {} failed with status {}",
+                        stage.name(),
+                        response.status()
+                    );
+                    return response;
+                }
+            }
+        }
+
+        match ctx.state.response.final_response {
+            Some(FinalResponse::Chat(response)) => {
+                Metrics::record_router_duration(
+                    metrics_labels::ROUTER_GRPC,
+                    self.backend_type,
+                    metrics_labels::CONNECTION_GRPC,
+                    &request_for_metrics.model,
+                    metrics_labels::ENDPOINT_CHAT,
+                    start.elapsed(),
+                );
+                axum::Json(response).into_response()
+            }
+            Some(FinalResponse::Generate(_))
+            | Some(FinalResponse::Embedding(_))
+            | Some(FinalResponse::Classify(_)) => {
+                error!(
+                    function = "execute_chat",
+                    "Wrong response type: expected Chat, got Generate/Embedding/Classify"
+                );
+                Metrics::record_router_error(
+                    metrics_labels::ROUTER_GRPC,
+                    self.backend_type,
+                    metrics_labels::CONNECTION_GRPC,
+                    &request_for_metrics.model,
+                    metrics_labels::ENDPOINT_CHAT,
+                    metrics_labels::ERROR_INTERNAL,
+                );
+                error::internal_error("wrong_response_type", "Internal error: wrong response type")
+            }
+            None => {
+                error!(
+                    function = "execute_chat",
+                    "No response produced by pipeline"
+                );
+                Metrics::record_router_error(
+                    metrics_labels::ROUTER_GRPC,
+                    self.backend_type,
+                    metrics_labels::CONNECTION_GRPC,
+                    &request_for_metrics.model,
+                    metrics_labels::ENDPOINT_CHAT,
+                    metrics_labels::ERROR_INTERNAL,
+                );
+                error::internal_error("no_response_produced", "No response produced")
+            }
+        }
+    }
+
+    /// Execute the complete pipeline for a generate request
+    pub async fn execute_generate(
+        &self,
+        request: Arc<GenerateRequest>,
+        headers: Option<http::HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Response {
+        let start = Instant::now();
+        let streaming = request.stream;
+
+        // Record request start
+        Metrics::record_router_request(
+            metrics_labels::ROUTER_GRPC,
+            self.backend_type,
+            metrics_labels::CONNECTION_GRPC,
+            model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+            metrics_labels::ENDPOINT_GENERATE,
+            bool_to_static_str(streaming),
+        );
+
+        let mut ctx = RequestContext::for_generate(request, headers, model_id.clone(), components);
+
+        for stage in self.stages.iter() {
+            match stage.execute(&mut ctx).await {
+                Ok(Some(response)) => {
+                    Metrics::record_router_duration(
+                        metrics_labels::ROUTER_GRPC,
+                        self.backend_type,
+                        metrics_labels::CONNECTION_GRPC,
+                        model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                        metrics_labels::ENDPOINT_GENERATE,
+                        start.elapsed(),
+                    );
+                    return response;
+                }
+                Ok(None) => continue,
+                Err(response) => {
+                    Metrics::record_router_error(
+                        metrics_labels::ROUTER_GRPC,
+                        self.backend_type,
+                        metrics_labels::CONNECTION_GRPC,
+                        model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                        metrics_labels::ENDPOINT_GENERATE,
+                        error_type_from_status(response.status()),
+                    );
+                    error!(
+                        "Stage {} failed with status {}",
+                        stage.name(),
+                        response.status()
+                    );
+                    return response;
+                }
+            }
+        }
+
+        match ctx.state.response.final_response {
+            Some(FinalResponse::Generate(response)) => {
+                Metrics::record_router_duration(
+                    metrics_labels::ROUTER_GRPC,
+                    self.backend_type,
+                    metrics_labels::CONNECTION_GRPC,
+                    model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                    metrics_labels::ENDPOINT_GENERATE,
+                    start.elapsed(),
+                );
+                axum::Json(response).into_response()
+            }
+            Some(FinalResponse::Chat(_))
+            | Some(FinalResponse::Embedding(_))
+            | Some(FinalResponse::Classify(_)) => {
+                error!(
+                    function = "execute_generate",
+                    "Wrong response type: expected Generate, got Chat/Embedding/Classify"
+                );
+                Metrics::record_router_error(
+                    metrics_labels::ROUTER_GRPC,
+                    self.backend_type,
+                    metrics_labels::CONNECTION_GRPC,
+                    model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                    metrics_labels::ENDPOINT_GENERATE,
+                    metrics_labels::ERROR_INTERNAL,
+                );
+                error::internal_error("wrong_response_type", "Internal error: wrong response type")
+            }
+            None => {
+                error!(
+                    function = "execute_generate",
+                    "No response produced by pipeline"
+                );
+                Metrics::record_router_error(
+                    metrics_labels::ROUTER_GRPC,
+                    self.backend_type,
+                    metrics_labels::CONNECTION_GRPC,
+                    model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                    metrics_labels::ENDPOINT_GENERATE,
+                    metrics_labels::ERROR_INTERNAL,
+                );
+                error::internal_error("no_response_produced", "No response produced")
+            }
+        }
+    }
+
+    /// Execute the complete pipeline for an embedding request
+    pub async fn execute_embeddings(
+        &self,
+        request: Arc<EmbeddingRequest>,
+        headers: Option<http::HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Response {
+        debug!(
+            "execute_embeddings: Starting execution for model: {}",
+            model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID)
+        );
+        let start = Instant::now();
+
+        // Record request start
+        Metrics::record_router_request(
+            metrics_labels::ROUTER_GRPC,
+            self.backend_type,
+            metrics_labels::CONNECTION_GRPC,
+            model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+            metrics_labels::ENDPOINT_EMBEDDINGS,
+            bool_to_static_str(false),
+        );
+
+        let mut ctx = RequestContext::for_embedding(request, headers, model_id.clone(), components);
+
+        for stage in self.stages.iter() {
+            debug!("execute_embeddings: Executing stage: {}", stage.name());
+            match stage.execute(&mut ctx).await {
+                Ok(Some(response)) => {
+                    debug!(
+                        "execute_embeddings: Stage {} returned final response.",
+                        stage.name()
+                    );
+                    Metrics::record_router_duration(
+                        metrics_labels::ROUTER_GRPC,
+                        self.backend_type,
+                        metrics_labels::CONNECTION_GRPC,
+                        model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                        metrics_labels::ENDPOINT_EMBEDDINGS,
+                        start.elapsed(),
+                    );
+                    return response;
+                }
+                Ok(None) => {
+                    debug!(
+                        "execute_embeddings: Stage {} completed, continuing to next stage.",
+                        stage.name()
+                    );
+                    continue;
+                }
+                Err(response) => {
+                    error!(
+                        "execute_embeddings: Stage {} failed with status {:?}, returning error response.",
+                        stage.name(),
+                        response.status()
+                    );
+                    Metrics::record_router_error(
+                        metrics_labels::ROUTER_GRPC,
+                        self.backend_type,
+                        metrics_labels::CONNECTION_GRPC,
+                        model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                        metrics_labels::ENDPOINT_EMBEDDINGS,
+                        error_type_from_status(response.status()),
+                    );
+                    return response;
+                }
+            }
+        }
+
+        debug!(
+            "execute_embeddings: Pipeline finished, processing final_response. Current state: {:?}",
+            ctx.state.response.final_response
+        );
+        match ctx.state.response.final_response {
+            Some(FinalResponse::Embedding(response)) => {
+                Metrics::record_router_duration(
+                    metrics_labels::ROUTER_GRPC,
+                    self.backend_type,
+                    metrics_labels::CONNECTION_GRPC,
+                    model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                    metrics_labels::ENDPOINT_EMBEDDINGS,
+                    start.elapsed(),
+                );
+                axum::Json(response).into_response()
+            }
+            Some(_) => {
+                error!(function = "execute_embeddings", "Wrong response type");
+                error::internal_error("wrong_response_type", "Internal error: wrong response type")
+            }
+            None => {
+                error!(
+                    function = "execute_embeddings",
+                    "No final response produced by pipeline."
+                );
+                error::internal_error("no_response_produced", "No response produced")
+            }
+        }
+    }
+
+    /// Execute the complete pipeline for a classify request
+    pub async fn execute_classify(
+        &self,
+        request: Arc<ClassifyRequest>,
+        headers: Option<http::HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Response {
+        debug!(
+            "execute_classify: Starting execution for model: {}",
+            model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID)
+        );
+        let start = Instant::now();
+
+        // Record request start
+        Metrics::record_router_request(
+            metrics_labels::ROUTER_GRPC,
+            self.backend_type,
+            metrics_labels::CONNECTION_GRPC,
+            model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+            metrics_labels::ENDPOINT_CLASSIFY,
+            bool_to_static_str(false), // Classify is never streaming
+        );
+
+        let mut ctx = RequestContext::for_classify(request, headers, model_id.clone(), components);
+
+        for stage in self.stages.iter() {
+            debug!("execute_classify: Executing stage: {}", stage.name());
+            match stage.execute(&mut ctx).await {
+                Ok(Some(response)) => {
+                    debug!(
+                        "execute_classify: Stage {} returned final response.",
+                        stage.name()
+                    );
+                    Metrics::record_router_duration(
+                        metrics_labels::ROUTER_GRPC,
+                        self.backend_type,
+                        metrics_labels::CONNECTION_GRPC,
+                        model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                        metrics_labels::ENDPOINT_CLASSIFY,
+                        start.elapsed(),
+                    );
+                    return response;
+                }
+                Ok(None) => {
+                    debug!(
+                        "execute_classify: Stage {} completed, continuing to next stage.",
+                        stage.name()
+                    );
+                    continue;
+                }
+                Err(response) => {
+                    error!(
+                        "execute_classify: Stage {} failed with status {:?}, returning error response.",
+                        stage.name(),
+                        response.status()
+                    );
+                    Metrics::record_router_error(
+                        metrics_labels::ROUTER_GRPC,
+                        self.backend_type,
+                        metrics_labels::CONNECTION_GRPC,
+                        model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                        metrics_labels::ENDPOINT_CLASSIFY,
+                        error_type_from_status(response.status()),
+                    );
+                    return response;
+                }
+            }
+        }
+
+        debug!(
+            "execute_classify: Pipeline finished, processing final_response. Current state: {:?}",
+            ctx.state.response.final_response
+        );
+        match ctx.state.response.final_response {
+            Some(FinalResponse::Classify(response)) => {
+                Metrics::record_router_duration(
+                    metrics_labels::ROUTER_GRPC,
+                    self.backend_type,
+                    metrics_labels::CONNECTION_GRPC,
+                    model_id.as_deref().unwrap_or(UNKNOWN_MODEL_ID),
+                    metrics_labels::ENDPOINT_CLASSIFY,
+                    start.elapsed(),
+                );
+                axum::Json(response).into_response()
+            }
+            Some(_) => {
+                error!(function = "execute_classify", "Wrong response type");
+                error::internal_error("wrong_response_type", "Internal error: wrong response type")
+            }
+            None => {
+                error!(
+                    function = "execute_classify",
+                    "No final response produced by pipeline."
+                );
+                error::internal_error("no_response_produced", "No response produced")
+            }
+        }
+    }
+
+    /// Execute chat pipeline for responses endpoint
+    ///
+    /// Used by ALL non-streaming /v1/responses requests.
+    /// Uses the same 7 pipeline stages as execute_chat(), with two differences:
+    /// 1. Returns Result<ChatCompletionResponse, Response> for tool_loop composition
+    /// 2. Disallows streaming (responses endpoint uses different SSE format)
+    pub async fn execute_chat_for_responses(
+        &self,
+        request: Arc<ChatCompletionRequest>,
+        headers: Option<http::HeaderMap>,
+        model_id: Option<String>,
+        components: Arc<SharedComponents>,
+    ) -> Result<ChatCompletionResponse, Response> {
+        let mut ctx = RequestContext::for_chat(request, headers, model_id, components);
+
+        for (idx, stage) in self.stages.iter().enumerate() {
+            match stage.execute(&mut ctx).await {
+                Ok(Some(_response)) => {
+                    // Streaming not supported for responses sync mode
+                    error!(
+                        function = "execute_chat_for_responses",
+                        "Streaming attempted in responses context"
+                    );
+                    return Err(error::bad_request(
+                        "streaming_not_supported",
+                        "Streaming is not supported in this context".to_string(),
+                    ));
+                }
+                Ok(None) => {
+                    continue;
+                }
+                Err(response) => {
+                    // Error occurred - return the response as-is to preserve HTTP status codes
+                    error!(
+                        "Stage {} ({}) failed with status {}",
+                        idx + 1,
+                        stage.name(),
+                        response.status()
+                    );
+                    return Err(response);
+                }
+            }
+        }
+
+        match ctx.state.response.final_response {
+            Some(FinalResponse::Chat(response)) => Ok(response),
+            Some(FinalResponse::Generate(_))
+            | Some(FinalResponse::Embedding(_))
+            | Some(FinalResponse::Classify(_)) => {
+                error!(
+                    function = "execute_chat_for_responses",
+                    "Wrong response type: expected Chat, got Generate/Embedding/Classify"
+                );
+                Err(error::internal_error(
+                    "wrong_response_type",
+                    "Internal error: wrong response type",
+                ))
+            }
+            None => {
+                error!(
+                    function = "execute_chat_for_responses",
+                    "No response produced by pipeline"
+                );
+                Err(error::internal_error(
+                    "no_response_produced",
+                    "No response produced",
+                ))
+            }
+        }
+    }
+
+    /// Execute Harmony Responses API request through all pipeline stages
+    ///
+    /// This method runs a single iteration of the Responses API request,
+    /// returning either ToolCallsFound (continue serving) or Completed (final response).
+    ///
+    /// Called by harmony::responses::serve_harmony_responses() for each iteration.
+    ///
+    /// # Arguments
+    ///
+    /// * `request` - Responses API request
+    /// * `ctx` - Harmony Responses context with MCP manager and components
+    ///
+    /// # Returns
+    ///
+    /// ResponsesIterationResult indicating whether to continue iteration or return
+    pub async fn execute_harmony_responses(
+        &self,
+        request: &crate::protocols::responses::ResponsesRequest,
+        harmony_ctx: &ResponsesContext,
+    ) -> Result<harmony::ResponsesIterationResult, Response> {
+        // Create RequestContext for this Responses request
+        let mut ctx = RequestContext::for_responses(
+            Arc::new(request.clone()),
+            None, // No headers needed for internal pipeline execution
+            None, // Model ID already set in request
+            harmony_ctx.components.clone(),
+        );
+
+        for (idx, stage) in self.stages.iter().enumerate() {
+            match stage.execute(&mut ctx).await {
+                Ok(Some(response)) => {
+                    // Stage returned early response (e.g., streaming) - not expected for Responses iteration
+                    error!(
+                        "Stage {} ({}) returned unexpected response during Responses iteration",
+                        idx + 1,
+                        stage.name()
+                    );
+                    return Err(response);
+                }
+                Ok(None) => {
+                    continue;
+                }
+                Err(response) => {
+                    // Stage failed
+                    error!(
+                        "Stage {} ({}) failed with status {}",
+                        idx + 1,
+                        stage.name(),
+                        response.status()
+                    );
+                    return Err(response);
+                }
+            }
+        }
+
+        // Extract ResponsesIterationResult from context
+        // This should have been set by HarmonyResponseProcessingStage
+        ctx.state
+            .response
+            .responses_iteration_result
+            .take()
+            .ok_or_else(|| {
+                error!(
+                    function = "execute_harmony_responses",
+                    "No ResponsesIterationResult produced by pipeline"
+                );
+                error::internal_error(
+                    "no_responses_iteration_result",
+                    "No ResponsesIterationResult produced by pipeline",
+                )
+            })
+    }
+
+    /// Execute Harmony Responses pipeline iteration with streaming support
+    ///
+    /// This version executes the pipeline up to the dispatch stage and returns
+    /// the raw ExecutionResult (with stream) and LoadGuards for token-level streaming processing.
+    /// The caller is responsible for keeping load_guards alive until stream processing completes.
+    pub async fn execute_harmony_responses_streaming(
+        &self,
+        request: &crate::protocols::responses::ResponsesRequest,
+        harmony_ctx: &ResponsesContext,
+    ) -> Result<(ExecutionResult, Option<LoadGuards>), Response> {
+        // Create RequestContext for this Responses request
+        let mut ctx = RequestContext::for_responses(
+            Arc::new(request.clone()),
+            None,
+            None,
+            harmony_ctx.components.clone(),
+        );
+
+        for (idx, stage) in self.stages.iter().enumerate() {
+            match stage.execute(&mut ctx).await {
+                Ok(Some(response)) => {
+                    error!(
+                        "Stage {} ({}) returned unexpected response during streaming Responses",
+                        idx + 1,
+                        stage.name()
+                    );
+                    return Err(response);
+                }
+                Ok(None) => continue,
+                Err(response) => {
+                    error!(
+                        "Stage {} ({}) failed with status {}",
+                        idx + 1,
+                        stage.name(),
+                        response.status()
+                    );
+                    return Err(response);
+                }
+            }
+        }
+
+        // Extract execution_result (the raw stream from workers) and load_guards
+        let execution_result = ctx.state.response.execution_result.take().ok_or_else(|| {
+            error!(
+                function = "execute_harmony_responses_streaming",
+                "No ExecutionResult produced by pipeline"
+            );
+            error::internal_error(
+                "no_execution_result_produced",
+                "No ExecutionResult produced by pipeline",
+            )
+        })?;
+
+        let load_guards = ctx.state.load_guards.take();
+
+        Ok((execution_result, load_guards))
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/proto_wrapper.rs

@@ -0,0 +1,520 @@
+//! Protocol buffer type wrappers for SGLang and vLLM backends
+//!
+//! This module provides unified enums that wrap proto types from both SGLang and vLLM,
+//! allowing the router to work with either backend transparently.
+
+use futures_util::StreamExt;
+use smg_grpc_client::{
+    sglang_proto::{self as sglang, generate_complete::MatchedStop},
+    sglang_scheduler::AbortOnDropStream as SglangStream,
+    vllm_engine::AbortOnDropStream as VllmStream,
+    vllm_proto as vllm,
+};
+
+/// Unified ProtoRequest
+#[derive(Clone)]
+pub enum ProtoRequest {
+    Generate(ProtoGenerateRequest),
+    Embed(ProtoEmbedRequest),
+}
+
+impl ProtoRequest {
+    /// Get request ID from either variant
+    pub fn request_id(&self) -> &str {
+        match self {
+            Self::Generate(req) => req.request_id(),
+            Self::Embed(req) => req.request_id(),
+        }
+    }
+}
+
+/// Unified GenerateRequest that works with both backends
+#[derive(Clone)]
+pub enum ProtoGenerateRequest {
+    Sglang(Box<sglang::GenerateRequest>),
+    Vllm(Box<vllm::GenerateRequest>),
+}
+
+impl ProtoGenerateRequest {
+    /// Get SGLang variant (panics if vLLM)
+    pub fn as_sglang(&self) -> &sglang::GenerateRequest {
+        match self {
+            Self::Sglang(req) => req,
+            Self::Vllm(_) => panic!("Expected SGLang GenerateRequest, got vLLM"),
+        }
+    }
+
+    /// Get mutable SGLang variant (panics if vLLM)
+    pub fn as_sglang_mut(&mut self) -> &mut sglang::GenerateRequest {
+        match self {
+            Self::Sglang(req) => req,
+            Self::Vllm(_) => panic!("Expected SGLang GenerateRequest, got vLLM"),
+        }
+    }
+
+    /// Get vLLM variant (panics if SGLang)
+    pub fn as_vllm(&self) -> &vllm::GenerateRequest {
+        match self {
+            Self::Vllm(req) => req,
+            Self::Sglang(_) => panic!("Expected vLLM GenerateRequest, got SGLang"),
+        }
+    }
+
+    /// Get mutable vLLM variant (panics if SGLang)
+    pub fn as_vllm_mut(&mut self) -> &mut vllm::GenerateRequest {
+        match self {
+            Self::Vllm(req) => req,
+            Self::Sglang(_) => panic!("Expected vLLM GenerateRequest, got SGLang"),
+        }
+    }
+
+    /// Check if this is SGLang
+    pub fn is_sglang(&self) -> bool {
+        matches!(self, Self::Sglang(_))
+    }
+
+    /// Check if this is vLLM
+    pub fn is_vllm(&self) -> bool {
+        matches!(self, Self::Vllm(_))
+    }
+
+    /// Clone the inner request (for passing to generate())
+    pub fn clone_inner(&self) -> Self {
+        self.clone()
+    }
+
+    /// Get request ID
+    pub fn request_id(&self) -> &str {
+        match self {
+            Self::Sglang(req) => &req.request_id,
+            Self::Vllm(req) => &req.request_id,
+        }
+    }
+}
+
+/// Unified GenerateResponse from stream
+pub enum ProtoGenerateResponse {
+    Sglang(Box<sglang::GenerateResponse>),
+    Vllm(vllm::GenerateResponse),
+}
+
+impl ProtoGenerateResponse {
+    /// Get the response variant (chunk, complete, or error)
+    ///
+    /// Consumes self to avoid cloning large proto messages in hot streaming path
+    pub fn into_response(self) -> ProtoResponseVariant {
+        match self {
+            Self::Sglang(resp) => match resp.response {
+                Some(sglang::generate_response::Response::Chunk(chunk)) => {
+                    ProtoResponseVariant::Chunk(ProtoGenerateStreamChunk::Sglang(chunk))
+                }
+                Some(sglang::generate_response::Response::Complete(complete)) => {
+                    ProtoResponseVariant::Complete(ProtoGenerateComplete::Sglang(complete))
+                }
+                Some(sglang::generate_response::Response::Error(error)) => {
+                    ProtoResponseVariant::Error(ProtoGenerateError::Sglang(error))
+                }
+                None => ProtoResponseVariant::None,
+            },
+            Self::Vllm(resp) => match resp.response {
+                Some(vllm::generate_response::Response::Chunk(chunk)) => {
+                    ProtoResponseVariant::Chunk(ProtoGenerateStreamChunk::Vllm(chunk))
+                }
+                Some(vllm::generate_response::Response::Complete(complete)) => {
+                    ProtoResponseVariant::Complete(ProtoGenerateComplete::Vllm(complete))
+                }
+                // Note: vLLM proto no longer has Error variant in GenerateResponse
+                None => ProtoResponseVariant::None,
+            },
+        }
+    }
+}
+
+/// Response variant extracted from GenerateResponse
+pub enum ProtoResponseVariant {
+    Chunk(ProtoGenerateStreamChunk),
+    Complete(ProtoGenerateComplete),
+    Error(ProtoGenerateError),
+    None,
+}
+
+/// Unified GenerateStreamChunk
+#[derive(Clone)]
+pub enum ProtoGenerateStreamChunk {
+    Sglang(sglang::GenerateStreamChunk),
+    Vllm(vllm::GenerateStreamChunk),
+}
+
+impl ProtoGenerateStreamChunk {
+    /// Get SGLang variant (panics if vLLM)
+    pub fn as_sglang(&self) -> &sglang::GenerateStreamChunk {
+        match self {
+            Self::Sglang(chunk) => chunk,
+            Self::Vllm(_) => panic!("Expected SGLang GenerateStreamChunk, got vLLM"),
+        }
+    }
+
+    /// Get vLLM variant (panics if SGLang)
+    pub fn as_vllm(&self) -> &vllm::GenerateStreamChunk {
+        match self {
+            Self::Vllm(chunk) => chunk,
+            Self::Sglang(_) => panic!("Expected vLLM GenerateStreamChunk, got SGLang"),
+        }
+    }
+
+    /// Check if this is SGLang
+    pub fn is_sglang(&self) -> bool {
+        matches!(self, Self::Sglang(_))
+    }
+
+    /// Check if this is vLLM
+    pub fn is_vllm(&self) -> bool {
+        matches!(self, Self::Vllm(_))
+    }
+
+    /// Get token IDs from chunk (common field)
+    pub fn token_ids(&self) -> &[u32] {
+        match self {
+            Self::Sglang(c) => &c.token_ids,
+            Self::Vllm(c) => &c.token_ids,
+        }
+    }
+
+    /// Get index (for n>1 support)
+    /// vLLM doesn't support n>1, so always returns 0
+    pub fn index(&self) -> u32 {
+        match self {
+            Self::Sglang(c) => c.index,
+            Self::Vllm(_) => 0, // vLLM doesn't support n>1
+        }
+    }
+
+    /// Get output logprobs (SGLang only, returns None for vLLM)
+    pub fn output_logprobs(&self) -> Option<&sglang::OutputLogProbs> {
+        match self {
+            Self::Sglang(c) => c.output_logprobs.as_ref(),
+            Self::Vllm(_) => None, // TODO: vLLM logprobs mapping
+        }
+    }
+
+    /// Get prompt tokens (cumulative)
+    pub fn prompt_tokens(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.prompt_tokens,
+            Self::Vllm(c) => c.prompt_tokens as i32,
+        }
+    }
+
+    /// Get completion tokens (cumulative)
+    pub fn completion_tokens(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.completion_tokens,
+            Self::Vllm(c) => c.completion_tokens as i32,
+        }
+    }
+
+    /// Get cached tokens (cumulative)
+    pub fn cached_tokens(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.cached_tokens,
+            Self::Vllm(c) => c.cached_tokens as i32,
+        }
+    }
+}
+
+/// Unified GenerateComplete response
+#[derive(Clone)]
+pub enum ProtoGenerateComplete {
+    Sglang(sglang::GenerateComplete),
+    Vllm(vllm::GenerateComplete),
+}
+
+impl ProtoGenerateComplete {
+    /// Get SGLang variant (panics if vLLM)
+    pub fn as_sglang(&self) -> &sglang::GenerateComplete {
+        match self {
+            Self::Sglang(complete) => complete,
+            Self::Vllm(_) => panic!("Expected SGLang GenerateComplete, got vLLM"),
+        }
+    }
+
+    /// Get mutable SGLang variant (panics if vLLM)
+    pub fn as_sglang_mut(&mut self) -> &mut sglang::GenerateComplete {
+        match self {
+            Self::Sglang(complete) => complete,
+            Self::Vllm(_) => panic!("Expected SGLang GenerateComplete, got vLLM"),
+        }
+    }
+
+    /// Get vLLM variant (panics if SGLang)
+    pub fn as_vllm(&self) -> &vllm::GenerateComplete {
+        match self {
+            Self::Vllm(complete) => complete,
+            Self::Sglang(_) => panic!("Expected vLLM GenerateComplete, got SGLang"),
+        }
+    }
+
+    /// Check if this is SGLang
+    pub fn is_sglang(&self) -> bool {
+        matches!(self, Self::Sglang(_))
+    }
+
+    /// Check if this is vLLM
+    pub fn is_vllm(&self) -> bool {
+        matches!(self, Self::Vllm(_))
+    }
+
+    /// Get token IDs from either backend (output_ids in proto)
+    pub fn token_ids(&self) -> &[u32] {
+        match self {
+            Self::Sglang(c) => &c.output_ids,
+            Self::Vllm(c) => &c.output_ids,
+        }
+    }
+
+    /// Get prompt tokens
+    pub fn prompt_tokens(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.prompt_tokens,
+            Self::Vllm(c) => c.prompt_tokens as i32,
+        }
+    }
+
+    /// Get completion tokens
+    pub fn completion_tokens(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.completion_tokens,
+            Self::Vllm(c) => c.completion_tokens as i32,
+        }
+    }
+
+    /// Get finish reason
+    pub fn finish_reason(&self) -> &str {
+        match self {
+            Self::Sglang(c) => &c.finish_reason,
+            Self::Vllm(c) => &c.finish_reason,
+        }
+    }
+
+    /// Get index (for n>1 support)
+    /// vLLM doesn't support n>1, so always returns 0
+    pub fn index(&self) -> u32 {
+        match self {
+            Self::Sglang(c) => c.index,
+            Self::Vllm(_) => 0, // vLLM doesn't have index field (n>1 not supported)
+        }
+    }
+
+    /// Get matched stop (SGLang only, returns oneof)
+    /// vLLM doesn't have matched_stop, returns None
+    pub fn matched_stop(&self) -> Option<&MatchedStop> {
+        match self {
+            Self::Sglang(c) => c.matched_stop.as_ref(),
+            Self::Vllm(_) => None, // vLLM doesn't have matched_stop
+        }
+    }
+
+    /// Get output IDs (decode tokens only)
+    pub fn output_ids(&self) -> &[u32] {
+        match self {
+            Self::Sglang(c) => &c.output_ids,
+            Self::Vllm(c) => &c.output_ids,
+        }
+    }
+
+    /// Get cached tokens
+    pub fn cached_tokens(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.cached_tokens,
+            Self::Vllm(c) => c.cached_tokens as i32,
+        }
+    }
+
+    /// Get input logprobs (SGLang only)
+    pub fn input_logprobs(&self) -> Option<&sglang::InputLogProbs> {
+        match self {
+            Self::Sglang(c) => c.input_logprobs.as_ref(),
+            Self::Vllm(_) => None, // vLLM doesn't have input_logprobs
+        }
+    }
+
+    /// Get output logprobs
+    pub fn output_logprobs(&self) -> Option<&sglang::OutputLogProbs> {
+        match self {
+            Self::Sglang(c) => c.output_logprobs.as_ref(),
+            Self::Vllm(_) => None, // TODO: vLLM logprobs mapping
+        }
+    }
+}
+
+/// Unified GenerateError
+/// Note: vLLM proto no longer has GenerateError - errors are returned via gRPC status
+#[derive(Clone)]
+pub enum ProtoGenerateError {
+    Sglang(sglang::GenerateError),
+}
+
+impl ProtoGenerateError {
+    /// Get error message
+    pub fn message(&self) -> &str {
+        match self {
+            Self::Sglang(e) => &e.message,
+        }
+    }
+}
+
+/// Unified stream wrapper
+pub enum ProtoStream {
+    Sglang(SglangStream),
+    Vllm(VllmStream),
+}
+
+impl ProtoStream {
+    /// Get next item from stream
+    pub async fn next(&mut self) -> Option<Result<ProtoGenerateResponse, tonic::Status>> {
+        match self {
+            Self::Sglang(stream) => stream
+                .next()
+                .await
+                .map(|result| result.map(|r| ProtoGenerateResponse::Sglang(Box::new(r)))),
+            Self::Vllm(stream) => stream
+                .next()
+                .await
+                .map(|result| result.map(ProtoGenerateResponse::Vllm)),
+        }
+    }
+
+    /// Mark stream as completed (no abort needed)
+    pub fn mark_completed(&mut self) {
+        match self {
+            Self::Sglang(stream) => stream.mark_completed(),
+            Self::Vllm(stream) => stream.mark_completed(),
+        }
+    }
+}
+
+/// Unified EmbedRequest that works with both backends
+#[derive(Clone)]
+pub enum ProtoEmbedRequest {
+    Sglang(Box<sglang::EmbedRequest>),
+}
+
+impl ProtoEmbedRequest {
+    /// Get SGLang variant
+    pub fn as_sglang(&self) -> &sglang::EmbedRequest {
+        match self {
+            Self::Sglang(req) => req,
+        }
+    }
+
+    /// Get mutable SGLang variant
+    pub fn as_sglang_mut(&mut self) -> &mut sglang::EmbedRequest {
+        match self {
+            Self::Sglang(req) => req,
+        }
+    }
+
+    /// Check if this is SGLang
+    pub fn is_sglang(&self) -> bool {
+        matches!(self, Self::Sglang(_))
+    }
+
+    /// Clone the inner request (for passing to embed())
+    pub fn clone_inner(&self) -> Self {
+        self.clone()
+    }
+
+    /// Get request ID
+    pub fn request_id(&self) -> &str {
+        match self {
+            Self::Sglang(req) => &req.request_id,
+        }
+    }
+}
+
+/// Unified EmbedResponse
+pub enum ProtoEmbedResponse {
+    Sglang(sglang::EmbedResponse),
+}
+
+impl ProtoEmbedResponse {
+    /// Get the response variant (complete or error)
+    pub fn into_response(self) -> ProtoEmbedResponseVariant {
+        match self {
+            Self::Sglang(resp) => match resp.response {
+                Some(sglang::embed_response::Response::Complete(complete)) => {
+                    ProtoEmbedResponseVariant::Complete(ProtoEmbedComplete::Sglang(complete))
+                }
+                Some(sglang::embed_response::Response::Error(error)) => {
+                    ProtoEmbedResponseVariant::Error(ProtoEmbedError::Sglang(error))
+                }
+                None => ProtoEmbedResponseVariant::None,
+            },
+        }
+    }
+}
+
+/// Response variant extracted from EmbedResponse
+pub enum ProtoEmbedResponseVariant {
+    Complete(ProtoEmbedComplete),
+    Error(ProtoEmbedError),
+    None,
+}
+
+/// Unified EmbedComplete response
+#[derive(Clone)]
+pub enum ProtoEmbedComplete {
+    Sglang(sglang::EmbedComplete),
+}
+
+impl ProtoEmbedComplete {
+    /// Get embeddings
+    pub fn embedding(&self) -> &[f32] {
+        match self {
+            Self::Sglang(c) => &c.embedding,
+        }
+    }
+
+    /// Get prompt tokens
+    pub fn prompt_tokens(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.prompt_tokens,
+        }
+    }
+
+    /// Get cached tokens
+    pub fn cached_tokens(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.cached_tokens,
+        }
+    }
+
+    /// Get embedding dimension
+    pub fn embedding_dim(&self) -> i32 {
+        match self {
+            Self::Sglang(c) => c.embedding_dim,
+        }
+    }
+}
+
+/// Unified EmbedError
+#[derive(Clone)]
+pub enum ProtoEmbedError {
+    Sglang(sglang::EmbedError),
+}
+
+impl ProtoEmbedError {
+    /// Get error message
+    pub fn message(&self) -> &str {
+        match self {
+            Self::Sglang(e) => &e.message,
+        }
+    }
+
+    /// Get error code
+    pub fn code(&self) -> &str {
+        match self {
+            Self::Sglang(e) => &e.code,
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/mod.rs

@@ -0,0 +1,9 @@
+//! Regular (non-harmony) model processing
+//!
+//! This module contains all code specific to regular tokenizer-based models,
+//! including pipeline stages, response processing, and streaming.
+
+pub(crate) mod processor;
+pub(crate) mod responses;
+pub(crate) mod stages;
+pub(crate) mod streaming;

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/processor.rs

@@ -0,0 +1,465 @@
+//! Shared response processing logic for gRPC routers
+//!
+//! This module contains response processing functions that are shared between
+//! the regular router and PD router.
+
+use std::{sync::Arc, time::Instant};
+
+use serde_json::Value;
+use smg_grpc_client::sglang_proto::generate_complete::MatchedStop;
+use tracing::error;
+
+use crate::{
+    protocols::{
+        chat::{ChatChoice, ChatCompletionMessage, ChatCompletionRequest, ChatCompletionResponse},
+        common::{FunctionCallResponse, ToolCall, ToolChoice, ToolChoiceValue},
+        generate::{GenerateMetaInfo, GenerateRequest, GenerateResponse},
+    },
+    reasoning_parser::ParserFactory as ReasoningParserFactory,
+    routers::{
+        error,
+        grpc::{
+            common::{response_collection, response_formatting},
+            context::{DispatchMetadata, ExecutionResult},
+            proto_wrapper::ProtoGenerateComplete,
+            utils,
+        },
+    },
+    tokenizer::{
+        stop::{SequenceDecoderOutput, StopSequenceDecoder},
+        traits::Tokenizer,
+    },
+    tool_parser::ParserFactory as ToolParserFactory,
+};
+
+/// Unified response processor for both routers
+#[derive(Clone)]
+pub(crate) struct ResponseProcessor {
+    pub tool_parser_factory: ToolParserFactory,
+    pub reasoning_parser_factory: ReasoningParserFactory,
+    pub configured_tool_parser: Option<String>,
+    pub configured_reasoning_parser: Option<String>,
+}
+
+impl ResponseProcessor {
+    pub fn new(
+        tool_parser_factory: ToolParserFactory,
+        reasoning_parser_factory: ReasoningParserFactory,
+        configured_tool_parser: Option<String>,
+        configured_reasoning_parser: Option<String>,
+    ) -> Self {
+        Self {
+            tool_parser_factory,
+            reasoning_parser_factory,
+            configured_tool_parser,
+            configured_reasoning_parser,
+        }
+    }
+
+    /// Process a single choice from GenerateComplete response
+    #[allow(clippy::too_many_arguments)]
+    pub async fn process_single_choice(
+        &self,
+        complete: &ProtoGenerateComplete,
+        index: usize,
+        original_request: &ChatCompletionRequest,
+        tokenizer: &Arc<dyn Tokenizer>,
+        stop_decoder: &mut StopSequenceDecoder,
+        history_tool_calls_count: usize,
+        reasoning_parser_available: bool,
+        tool_parser_available: bool,
+    ) -> Result<ChatChoice, String> {
+        stop_decoder.reset();
+        // Decode tokens
+        let outputs = stop_decoder
+            .process_tokens(complete.output_ids())
+            .map_err(|e| format!("Failed to process tokens: {}", e))?;
+
+        // Accumulate text with early breaks
+        let mut final_text = String::new();
+        for output in outputs {
+            match output {
+                SequenceDecoderOutput::Text(t) => final_text.push_str(&t),
+                SequenceDecoderOutput::StoppedWithText(t) => {
+                    final_text.push_str(&t);
+                    break;
+                }
+                SequenceDecoderOutput::Stopped => break,
+                SequenceDecoderOutput::Held => {}
+            }
+        }
+
+        // Flush remaining text
+        if let SequenceDecoderOutput::Text(t) = stop_decoder.flush() {
+            final_text.push_str(&t);
+        }
+
+        // Step 1: Handle reasoning content parsing
+        let mut reasoning_text: Option<String> = None;
+        let mut processed_text = final_text;
+
+        if original_request.separate_reasoning && reasoning_parser_available {
+            let pooled_parser = utils::get_reasoning_parser(
+                &self.reasoning_parser_factory,
+                self.configured_reasoning_parser.as_deref(),
+                &original_request.model,
+            );
+
+            let mut parser = pooled_parser.lock().await;
+            match parser.detect_and_parse_reasoning(&processed_text) {
+                Ok(result) => {
+                    if !result.reasoning_text.is_empty() {
+                        reasoning_text = Some(result.reasoning_text);
+                    }
+                    processed_text = result.normal_text;
+                }
+                Err(e) => {
+                    return Err(format!("Reasoning parsing error: {}", e));
+                }
+            }
+        }
+
+        // Step 2: Handle tool call parsing
+        let mut tool_calls: Option<Vec<ToolCall>> = None;
+        let tool_choice_enabled = !matches!(
+            &original_request.tool_choice,
+            Some(ToolChoice::Value(ToolChoiceValue::None))
+        );
+
+        if tool_choice_enabled && original_request.tools.is_some() {
+            // Check if JSON schema constraint was used (specific function or required mode)
+            let used_json_schema = match &original_request.tool_choice {
+                Some(ToolChoice::Function { .. }) => true,
+                Some(ToolChoice::Value(ToolChoiceValue::Required)) => true,
+                Some(ToolChoice::AllowedTools { mode, .. }) => mode == "required",
+                _ => false,
+            };
+
+            if used_json_schema {
+                (tool_calls, processed_text) = utils::parse_json_schema_response(
+                    &processed_text,
+                    &original_request.tool_choice,
+                    &original_request.model,
+                    history_tool_calls_count,
+                );
+            } else if tool_parser_available {
+                (tool_calls, processed_text) = self
+                    .parse_tool_calls(
+                        &processed_text,
+                        &original_request.model,
+                        history_tool_calls_count,
+                    )
+                    .await;
+            }
+        }
+
+        // Step 3: Use finish reason directly from proto (already OpenAI-compatible string)
+        let finish_reason_str = complete.finish_reason();
+
+        // Override finish reason if we have tool calls
+        let final_finish_reason_str = if tool_calls.is_some() {
+            "tool_calls"
+        } else {
+            finish_reason_str
+        };
+
+        // Extract matched_stop information from proto
+        let matched_stop = match complete.matched_stop() {
+            Some(MatchedStop::MatchedTokenId(token_id)) => {
+                Some(Value::Number(serde_json::Number::from(*token_id)))
+            }
+            Some(MatchedStop::MatchedStopStr(stop_str)) => Some(Value::String(stop_str.clone())),
+            None => None,
+        };
+
+        // Step 4: Convert output logprobs if present
+        let logprobs = if let Some(proto_logprobs) = complete.output_logprobs() {
+            match utils::convert_proto_to_openai_logprobs(proto_logprobs, tokenizer) {
+                Ok(logprobs) => Some(logprobs),
+                Err(e) => {
+                    error!("Failed to convert logprobs: {}", e);
+                    None
+                }
+            }
+        } else {
+            None
+        };
+
+        // Step 5: Build ChatCompletionMessage (proper response message type)
+        let chat_message = ChatCompletionMessage {
+            role: "assistant".to_string(),
+            content: if processed_text.is_empty() {
+                None
+            } else {
+                Some(processed_text)
+            },
+            tool_calls,
+            reasoning_content: reasoning_text,
+        };
+
+        // Step 6: Build ChatChoice
+        Ok(ChatChoice {
+            index: index as u32,
+            message: chat_message,
+            logprobs,
+            finish_reason: Some(final_finish_reason_str.to_string()),
+            matched_stop,
+            hidden_states: None,
+        })
+    }
+
+    /// Process non-streaming chat response (collects all responses and builds final response)
+    pub async fn process_non_streaming_chat_response(
+        &self,
+        execution_result: ExecutionResult,
+        chat_request: Arc<ChatCompletionRequest>,
+        dispatch: DispatchMetadata,
+        tokenizer: Arc<dyn Tokenizer>,
+        stop_decoder: &mut StopSequenceDecoder,
+        request_logprobs: bool,
+    ) -> Result<ChatCompletionResponse, axum::response::Response> {
+        // Collect all responses from the execution result
+        let all_responses =
+            response_collection::collect_responses(execution_result, request_logprobs).await?;
+
+        let history_tool_calls_count = utils::get_history_tool_calls_count(&chat_request);
+
+        // Check parser availability once upfront (not per choice)
+        let reasoning_parser_available = chat_request.separate_reasoning
+            && utils::check_reasoning_parser_availability(
+                &self.reasoning_parser_factory,
+                self.configured_reasoning_parser.as_deref(),
+                &chat_request.model,
+            );
+
+        let tool_choice_enabled = !matches!(
+            &chat_request.tool_choice,
+            Some(ToolChoice::Value(ToolChoiceValue::None))
+        );
+
+        let tool_parser_available = tool_choice_enabled
+            && chat_request.tools.is_some()
+            && utils::check_tool_parser_availability(
+                &self.tool_parser_factory,
+                self.configured_tool_parser.as_deref(),
+                &chat_request.model,
+            );
+
+        // Log once per request (not per choice)
+        if chat_request.separate_reasoning && !reasoning_parser_available {
+            tracing::debug!(
+                "No reasoning parser found for model '{}', skipping reasoning parsing",
+                chat_request.model
+            );
+        }
+
+        if chat_request.tools.is_some() && tool_choice_enabled && !tool_parser_available {
+            tracing::debug!(
+                "No tool parser found for model '{}', skipping tool call parsing",
+                chat_request.model
+            );
+        }
+
+        // Process all choices
+        let mut choices = Vec::new();
+        for (index, complete) in all_responses.iter().enumerate() {
+            match self
+                .process_single_choice(
+                    complete,
+                    index,
+                    &chat_request,
+                    &tokenizer,
+                    stop_decoder,
+                    history_tool_calls_count,
+                    reasoning_parser_available,
+                    tool_parser_available,
+                )
+                .await
+            {
+                Ok(choice) => choices.push(choice),
+                Err(e) => {
+                    return Err(error::internal_error(
+                        "process_choice_failed",
+                        format!("Failed to process choice {}: {}", index, e),
+                    ));
+                }
+            }
+        }
+
+        // Build usage
+        let usage = response_formatting::build_usage(&all_responses);
+
+        // Build final ChatCompletionResponse
+        Ok(
+            ChatCompletionResponse::builder(&dispatch.request_id, &dispatch.model)
+                .created(dispatch.created)
+                .choices(choices)
+                .usage(usage)
+                .maybe_system_fingerprint(dispatch.weight_version.clone())
+                .build(),
+        )
+    }
+
+    /// Parse tool calls using model-specific parser
+    pub async fn parse_tool_calls(
+        &self,
+        processed_text: &str,
+        model: &str,
+        history_tool_calls_count: usize,
+    ) -> (Option<Vec<ToolCall>>, String) {
+        // Get pooled parser for this model
+        let pooled_parser = utils::get_tool_parser(
+            &self.tool_parser_factory,
+            self.configured_tool_parser.as_deref(),
+            model,
+        );
+
+        // Try parsing directly (parser will handle detection internally)
+        let result = {
+            let parser = pooled_parser.lock().await;
+            parser.parse_complete(processed_text).await
+            // Lock is dropped here
+        };
+
+        match result {
+            Ok((normal_text, parsed_tool_calls)) => {
+                if parsed_tool_calls.is_empty() {
+                    return (None, normal_text);
+                }
+
+                let spec_tool_calls = parsed_tool_calls
+                    .into_iter()
+                    .enumerate()
+                    .map(|(index, tc)| {
+                        // Generate ID for this tool call
+                        let id = utils::generate_tool_call_id(
+                            model,
+                            &tc.function.name,
+                            index,
+                            history_tool_calls_count,
+                        );
+                        ToolCall {
+                            id,
+                            tool_type: "function".to_string(),
+                            function: FunctionCallResponse {
+                                name: tc.function.name,
+                                arguments: Some(tc.function.arguments),
+                            },
+                        }
+                    })
+                    .collect();
+                (Some(spec_tool_calls), normal_text)
+            }
+            Err(e) => {
+                error!("Tool call parsing error: {}", e);
+                (None, processed_text.to_string())
+            }
+        }
+    }
+
+    /// Process non-streaming generate response (collects all responses and builds final response array)
+    pub async fn process_non_streaming_generate_response(
+        &self,
+        execution_result: ExecutionResult,
+        _generate_request: Arc<GenerateRequest>,
+        dispatch: DispatchMetadata,
+        stop_decoder: &mut StopSequenceDecoder,
+        request_logprobs: bool,
+        start_time: Instant,
+    ) -> Result<Vec<GenerateResponse>, axum::response::Response> {
+        // Collect all responses from the execution result
+        let all_responses =
+            response_collection::collect_responses(execution_result, request_logprobs).await?;
+
+        // Process each completion
+        let mut result_array = Vec::new();
+        for complete in all_responses {
+            stop_decoder.reset();
+
+            // Process tokens through stop decoder
+            let outputs = match stop_decoder.process_tokens(complete.output_ids()) {
+                Ok(outputs) => outputs,
+                Err(e) => {
+                    return Err(error::internal_error(
+                        "process_tokens_failed",
+                        format!("Failed to process tokens: {}", e),
+                    ))
+                }
+            };
+
+            // Accumulate text with early breaks
+            let mut decoded_text = String::new();
+            for output in outputs {
+                match output {
+                    SequenceDecoderOutput::Text(t) => decoded_text.push_str(&t),
+                    SequenceDecoderOutput::StoppedWithText(t) => {
+                        decoded_text.push_str(&t);
+                        break;
+                    }
+                    SequenceDecoderOutput::Stopped => break,
+                    SequenceDecoderOutput::Held => {}
+                }
+            }
+
+            // Flush remaining text
+            if let SequenceDecoderOutput::Text(t) = stop_decoder.flush() {
+                decoded_text.push_str(&t);
+            }
+
+            let output_ids = complete.output_ids().to_vec();
+            let finish_reason_str = complete.finish_reason();
+
+            // Parse finish_reason from string to proper type
+            let finish_reason =
+                utils::parse_finish_reason(finish_reason_str, complete.completion_tokens());
+
+            // Handle matched_stop if present
+            let matched_stop = complete.matched_stop().map(|matched| match matched {
+                MatchedStop::MatchedTokenId(id) => serde_json::json!(id),
+                MatchedStop::MatchedStopStr(s) => serde_json::json!(s),
+            });
+
+            // Extract logprobs if requested (convert proto types to Generate format)
+            let input_token_logprobs = if request_logprobs {
+                complete
+                    .input_logprobs()
+                    .map(utils::convert_generate_input_logprobs)
+            } else {
+                None
+            };
+
+            let output_token_logprobs = if request_logprobs {
+                complete
+                    .output_logprobs()
+                    .map(utils::convert_generate_output_logprobs)
+            } else {
+                None
+            };
+
+            // Build GenerateResponse struct
+            let meta_info = GenerateMetaInfo {
+                id: dispatch.request_id.clone(),
+                finish_reason,
+                prompt_tokens: complete.prompt_tokens() as u32,
+                weight_version: dispatch
+                    .weight_version
+                    .clone()
+                    .unwrap_or_else(|| "default".to_string()),
+                input_token_logprobs,
+                output_token_logprobs,
+                completion_tokens: complete.completion_tokens() as u32,
+                cached_tokens: complete.cached_tokens() as u32,
+                e2e_latency: start_time.elapsed().as_secs_f64(),
+                matched_stop,
+            };
+
+            result_array.push(GenerateResponse {
+                text: decoded_text,
+                output_ids,
+                meta_info,
+            });
+        }
+
+        Ok(result_array)
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/responses/common.rs

@@ -0,0 +1,470 @@
+//! Shared helpers and state tracking for Regular Responses
+//!
+//! This module contains common utilities used by both streaming and non-streaming paths:
+//! - ToolLoopState for tracking multi-turn tool calling
+//! - Helper functions for tool preparation and extraction
+//! - MCP metadata builders
+//! - Conversation history loading
+
+use std::sync::Arc;
+
+use axum::response::Response;
+use data_connector::{self, ConversationId, ResponseId};
+use serde_json::{json, Value};
+use smg_mcp::{self as mcp, McpManager};
+use tracing::{debug, warn};
+use uuid::Uuid;
+
+use crate::{
+    protocols::{
+        chat::ChatCompletionRequest,
+        common::{Function, Tool, ToolChoice, ToolChoiceValue},
+        responses::{
+            self, McpToolInfo, ResponseContentPart, ResponseInput, ResponseInputOutputItem,
+            ResponseOutputItem, ResponsesRequest,
+        },
+    },
+    routers::{error, grpc::common::responses::ResponsesContext},
+};
+
+// ============================================================================
+// Tool Loop State
+// ============================================================================
+
+/// State for tracking multi-turn tool calling loop
+pub(super) struct ToolLoopState {
+    pub iteration: usize,
+    pub total_calls: usize,
+    pub conversation_history: Vec<ResponseInputOutputItem>,
+    pub original_input: ResponseInput,
+    pub mcp_call_items: Vec<ResponseOutputItem>,
+    pub server_label: String,
+}
+
+impl ToolLoopState {
+    pub fn new(original_input: ResponseInput, server_label: String) -> Self {
+        Self {
+            iteration: 0,
+            total_calls: 0,
+            conversation_history: Vec::new(),
+            original_input,
+            mcp_call_items: Vec::new(),
+            server_label,
+        }
+    }
+
+    pub fn record_call(
+        &mut self,
+        call_id: String,
+        tool_name: String,
+        args_json_str: String,
+        output_str: String,
+        success: bool,
+        error: Option<String>,
+    ) {
+        // Add function_tool_call item with both arguments and output
+        self.conversation_history
+            .push(ResponseInputOutputItem::FunctionToolCall {
+                id: call_id.clone(),
+                call_id: call_id.clone(),
+                name: tool_name.clone(),
+                arguments: args_json_str.clone(),
+                output: Some(output_str.clone()),
+                status: Some("completed".to_string()),
+            });
+
+        // Add mcp_call output item for metadata
+        let mcp_call = build_mcp_call_item(
+            &tool_name,
+            &args_json_str,
+            &output_str,
+            &self.server_label,
+            success,
+            error.as_deref(),
+        );
+        self.mcp_call_items.push(mcp_call);
+    }
+}
+
+// ============================================================================
+// Tool Preparation and Extraction
+// ============================================================================
+
+/// Merge function tools from request with MCP tools and set tool_choice based on iteration
+pub(super) fn prepare_chat_tools_and_choice(
+    chat_request: &mut ChatCompletionRequest,
+    mcp_chat_tools: &[Tool],
+    iteration: usize,
+) {
+    // Merge function tools from request with MCP tools
+    let mut all_tools = chat_request.tools.clone().unwrap_or_default();
+    all_tools.extend(mcp_chat_tools.iter().cloned());
+    chat_request.tools = Some(all_tools);
+
+    // Set tool_choice based on iteration
+    // - Iteration 0: Use user's tool_choice or default to auto
+    // - Iteration 1+: Always use auto to avoid infinite loops
+    chat_request.tool_choice = if iteration == 0 {
+        chat_request
+            .tool_choice
+            .clone()
+            .or(Some(ToolChoice::Value(ToolChoiceValue::Auto)))
+    } else {
+        Some(ToolChoice::Value(ToolChoiceValue::Auto))
+    };
+}
+
+/// Tool call extracted from a ChatCompletionResponse
+#[derive(Debug, Clone)]
+pub(super) struct ExtractedToolCall {
+    pub call_id: String,
+    pub name: String,
+    pub arguments: String,
+}
+
+/// Extract all tool calls from chat response (for parallel tool call support)
+pub(super) fn extract_all_tool_calls_from_chat(
+    response: &crate::protocols::chat::ChatCompletionResponse,
+) -> Vec<ExtractedToolCall> {
+    // Check if response has choices with tool calls
+    let Some(choice) = response.choices.first() else {
+        return Vec::new();
+    };
+    let message = &choice.message;
+
+    // Look for tool_calls in the message
+    if let Some(tool_calls) = &message.tool_calls {
+        tool_calls
+            .iter()
+            .map(|tool_call| ExtractedToolCall {
+                call_id: tool_call.id.clone(),
+                name: tool_call.function.name.clone(),
+                arguments: tool_call
+                    .function
+                    .arguments
+                    .clone()
+                    .unwrap_or_else(|| "{}".to_string()),
+            })
+            .collect()
+    } else {
+        Vec::new()
+    }
+}
+
+/// Convert MCP tools to Chat API tool format
+pub(super) fn convert_mcp_tools_to_chat_tools(mcp_tools: &[mcp::Tool]) -> Vec<Tool> {
+    mcp_tools
+        .iter()
+        .map(|tool_info| Tool {
+            tool_type: "function".to_string(),
+            function: Function {
+                name: tool_info.name.to_string(),
+                description: tool_info.description.as_ref().map(|d| d.to_string()),
+                parameters: Value::Object((*tool_info.input_schema).clone()),
+                strict: None,
+            },
+        })
+        .collect()
+}
+
+// ============================================================================
+// MCP Metadata Builders
+// ============================================================================
+
+/// Generate unique ID for MCP items
+pub(super) fn generate_mcp_id(prefix: &str) -> String {
+    format!("{}_{}", prefix, Uuid::new_v4())
+}
+
+/// Build mcp_list_tools output item
+pub(super) fn build_mcp_list_tools_item(
+    mcp: &Arc<McpManager>,
+    server_label: &str,
+    server_keys: &[String],
+) -> ResponseOutputItem {
+    let tools = mcp.list_tools_for_servers(server_keys);
+    let tools_info: Vec<McpToolInfo> = tools
+        .iter()
+        .map(|t| McpToolInfo {
+            name: t.name.to_string(),
+            description: t.description.as_ref().map(|d| d.to_string()),
+            input_schema: Value::Object((*t.input_schema).clone()),
+            annotations: Some(json!({
+                "read_only": false
+            })),
+        })
+        .collect();
+
+    ResponseOutputItem::McpListTools {
+        id: generate_mcp_id("mcpl"),
+        server_label: server_label.to_string(),
+        tools: tools_info,
+    }
+}
+
+/// Build mcp_call output item
+pub(super) fn build_mcp_call_item(
+    tool_name: &str,
+    arguments: &str,
+    output: &str,
+    server_label: &str,
+    success: bool,
+    error: Option<&str>,
+) -> ResponseOutputItem {
+    ResponseOutputItem::McpCall {
+        id: generate_mcp_id("mcp"),
+        status: if success { "completed" } else { "failed" }.to_string(),
+        approval_request_id: None,
+        arguments: arguments.to_string(),
+        error: error.map(|e| e.to_string()),
+        name: tool_name.to_string(),
+        output: output.to_string(),
+        server_label: server_label.to_string(),
+    }
+}
+
+// ============================================================================
+// Conversation History Loading
+// ============================================================================
+
+/// Load conversation history and response chains, returning modified request
+pub(super) async fn load_conversation_history(
+    ctx: &ResponsesContext,
+    request: &ResponsesRequest,
+) -> Result<ResponsesRequest, Response> {
+    let mut modified_request = request.clone();
+    let mut conversation_items: Option<Vec<ResponseInputOutputItem>> = None;
+
+    // Handle previous_response_id by loading response chain
+    if let Some(ref prev_id_str) = modified_request.previous_response_id {
+        let prev_id = ResponseId::from(prev_id_str.as_str());
+        match ctx
+            .response_storage
+            .get_response_chain(&prev_id, None)
+            .await
+        {
+            Ok(chain) => {
+                let mut items = Vec::new();
+                for stored in chain.responses.iter() {
+                    // Convert input items from stored input (which is now a JSON array)
+                    if let Some(input_arr) = stored.input.as_array() {
+                        for item in input_arr {
+                            match serde_json::from_value::<ResponseInputOutputItem>(item.clone()) {
+                                Ok(input_item) => {
+                                    items.push(input_item);
+                                }
+                                Err(e) => {
+                                    warn!(
+                                        "Failed to deserialize stored input item: {}. Item: {}",
+                                        e, item
+                                    );
+                                }
+                            }
+                        }
+                    }
+
+                    // Convert output items from stored output (which is now a JSON array)
+                    if let Some(output_arr) = stored.output.as_array() {
+                        for item in output_arr {
+                            match serde_json::from_value::<ResponseInputOutputItem>(item.clone()) {
+                                Ok(output_item) => {
+                                    items.push(output_item);
+                                }
+                                Err(e) => {
+                                    warn!(
+                                        "Failed to deserialize stored output item: {}. Item: {}",
+                                        e, item
+                                    );
+                                }
+                            }
+                        }
+                    }
+                }
+                conversation_items = Some(items);
+                modified_request.previous_response_id = None;
+            }
+            Err(e) => {
+                warn!(
+                    "Failed to load previous response chain for {}: {}",
+                    prev_id_str, e
+                );
+            }
+        }
+    }
+
+    // Handle conversation by loading conversation history
+    if let Some(ref conv_id_str) = request.conversation {
+        let conv_id = ConversationId::from(conv_id_str.as_str());
+
+        // Check if conversation exists - return error if not found
+        let conversation = ctx
+            .conversation_storage
+            .get_conversation(&conv_id)
+            .await
+            .map_err(|e| {
+                error::internal_error(
+                    "check_conversation_failed",
+                    format!("Failed to check conversation: {}", e),
+                )
+            })?;
+
+        if conversation.is_none() {
+            return Err(error::not_found(
+                "conversation_not_found",
+                format!(
+                    "Conversation '{}' not found. Please create the conversation first using the conversations API.",
+                    conv_id_str
+                )
+            ));
+        }
+
+        // Load conversation history
+        const MAX_CONVERSATION_HISTORY_ITEMS: usize = 100;
+        let params = data_connector::ListParams {
+            limit: MAX_CONVERSATION_HISTORY_ITEMS,
+            order: data_connector::SortOrder::Asc,
+            after: None,
+        };
+
+        match ctx
+            .conversation_item_storage
+            .list_items(&conv_id, params)
+            .await
+        {
+            Ok(stored_items) => {
+                let mut items: Vec<ResponseInputOutputItem> = Vec::new();
+                for item in stored_items.into_iter() {
+                    if item.item_type == "message" {
+                        if let Ok(content_parts) =
+                            serde_json::from_value::<Vec<ResponseContentPart>>(item.content.clone())
+                        {
+                            items.push(ResponseInputOutputItem::Message {
+                                id: item.id.0.clone(),
+                                role: item.role.clone().unwrap_or_else(|| "user".to_string()),
+                                content: content_parts,
+                                status: item.status.clone(),
+                            });
+                        }
+                    }
+                }
+
+                // Append current request
+                match &modified_request.input {
+                    ResponseInput::Text(text) => {
+                        items.push(ResponseInputOutputItem::Message {
+                            id: format!("msg_u_{}", conv_id.0),
+                            role: "user".to_string(),
+                            content: vec![ResponseContentPart::InputText { text: text.clone() }],
+                            status: Some("completed".to_string()),
+                        });
+                    }
+                    ResponseInput::Items(current_items) => {
+                        // Process all item types, converting SimpleInputMessage to Message
+                        for item in current_items.iter() {
+                            let normalized = responses::normalize_input_item(item);
+                            items.push(normalized);
+                        }
+                    }
+                }
+
+                modified_request.input = ResponseInput::Items(items);
+            }
+            Err(e) => {
+                warn!("Failed to load conversation history: {}", e);
+            }
+        }
+    }
+
+    // If we have conversation_items from previous_response_id, merge them
+    if let Some(mut items) = conversation_items {
+        // Append current request
+        match &modified_request.input {
+            ResponseInput::Text(text) => {
+                items.push(ResponseInputOutputItem::Message {
+                    id: format!(
+                        "msg_u_{}",
+                        request
+                            .previous_response_id
+                            .as_ref()
+                            .unwrap_or(&"new".to_string())
+                    ),
+                    role: "user".to_string(),
+                    content: vec![ResponseContentPart::InputText { text: text.clone() }],
+                    status: Some("completed".to_string()),
+                });
+            }
+            ResponseInput::Items(current_items) => {
+                // Process all item types, converting SimpleInputMessage to Message
+                for item in current_items.iter() {
+                    let normalized = responses::normalize_input_item(item);
+                    items.push(normalized);
+                }
+            }
+        }
+
+        modified_request.input = ResponseInput::Items(items);
+    }
+
+    debug!(
+        has_previous_response = request.previous_response_id.is_some(),
+        has_conversation = request.conversation.is_some(),
+        "Loaded conversation history"
+    );
+
+    Ok(modified_request)
+}
+
+/// Build next request with updated conversation history
+pub(super) fn build_next_request(
+    state: &ToolLoopState,
+    current_request: &ResponsesRequest,
+) -> ResponsesRequest {
+    // Start with original input
+    let mut input_items = match &state.original_input {
+        ResponseInput::Text(text) => vec![ResponseInputOutputItem::Message {
+            id: format!("msg_u_{}", state.iteration),
+            role: "user".to_string(),
+            content: vec![ResponseContentPart::InputText { text: text.clone() }],
+            status: Some("completed".to_string()),
+        }],
+        ResponseInput::Items(items) => items.iter().map(responses::normalize_input_item).collect(),
+    };
+
+    // Append all conversation history (function calls and outputs)
+    input_items.extend_from_slice(&state.conversation_history);
+
+    // Build new request for next iteration
+    ResponsesRequest {
+        input: ResponseInput::Items(input_items),
+        model: current_request.model.clone(),
+        instructions: current_request.instructions.clone(),
+        tools: current_request.tools.clone(),
+        max_output_tokens: current_request.max_output_tokens,
+        temperature: current_request.temperature,
+        top_p: current_request.top_p,
+        stream: current_request.stream,
+        store: Some(false), // Don't store intermediate responses
+        background: Some(false),
+        max_tool_calls: current_request.max_tool_calls,
+        tool_choice: current_request.tool_choice.clone(),
+        parallel_tool_calls: current_request.parallel_tool_calls,
+        previous_response_id: None,
+        conversation: None,
+        user: current_request.user.clone(),
+        metadata: current_request.metadata.clone(),
+        include: current_request.include.clone(),
+        reasoning: current_request.reasoning.clone(),
+        service_tier: current_request.service_tier.clone(),
+        top_logprobs: current_request.top_logprobs,
+        truncation: current_request.truncation.clone(),
+        text: current_request.text.clone(),
+        request_id: None,
+        priority: current_request.priority,
+        frequency_penalty: current_request.frequency_penalty,
+        presence_penalty: current_request.presence_penalty,
+        stop: current_request.stop.clone(),
+        top_k: current_request.top_k,
+        min_p: current_request.min_p,
+        repetition_penalty: current_request.repetition_penalty,
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/responses/conversions.rs

@@ -0,0 +1,428 @@
+//! Conversion utilities for translating between /v1/responses and /v1/chat/completions formats
+//!
+//! This module implements the conversion approach where:
+//! 1. ResponsesRequest → ChatCompletionRequest (for backend processing)
+//! 2. ChatCompletionResponse → ResponsesResponse (for client response)
+//!
+//! This allows the gRPC router to reuse the existing chat pipeline infrastructure
+//! without requiring Python backend changes.
+
+use crate::{
+    protocols::{
+        chat::{ChatCompletionRequest, ChatCompletionResponse, ChatMessage, MessageContent},
+        common::{
+            FunctionCallResponse, JsonSchemaFormat, ResponseFormat, StreamOptions, ToolCall,
+            UsageInfo,
+        },
+        responses::{
+            ResponseContentPart, ResponseInput, ResponseInputOutputItem, ResponseOutputItem,
+            ResponseReasoningContent::ReasoningText, ResponseStatus, ResponsesRequest,
+            ResponsesResponse, ResponsesUsage, StringOrContentParts, TextConfig, TextFormat,
+        },
+        UNKNOWN_MODEL_ID,
+    },
+    routers::grpc::common::responses::utils::extract_tools_from_response_tools,
+};
+
+/// Convert a ResponsesRequest to ChatCompletionRequest for processing through the chat pipeline
+///
+/// # Conversion Logic
+/// - `input` (text/items) → `messages` (chat messages)
+/// - `instructions` → system message (prepended)
+/// - `max_output_tokens` → `max_completion_tokens`
+/// - `tools` → function tools extracted from ResponseTools
+/// - `tool_choice` → passed through from request
+/// - Response-specific fields (previous_response_id, conversation) are handled by router
+pub(crate) fn responses_to_chat(req: &ResponsesRequest) -> Result<ChatCompletionRequest, String> {
+    let mut messages = Vec::new();
+
+    // 1. Add system message if instructions provided
+    if let Some(instructions) = &req.instructions {
+        messages.push(ChatMessage::System {
+            content: MessageContent::Text(instructions.clone()),
+            name: None,
+        });
+    }
+
+    // 2. Convert input to chat messages
+    match &req.input {
+        ResponseInput::Text(text) => {
+            // Simple text input → user message
+            messages.push(ChatMessage::User {
+                content: MessageContent::Text(text.clone()),
+                name: None,
+            });
+        }
+        ResponseInput::Items(items) => {
+            // Structured items → convert each to appropriate chat message
+            for item in items {
+                match item {
+                    ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => {
+                        // Convert SimpleInputMessage to chat message
+                        let text = match content {
+                            StringOrContentParts::String(s) => s.clone(),
+                            StringOrContentParts::Array(parts) => {
+                                // Extract text from content parts (only InputText supported)
+                                parts
+                                    .iter()
+                                    .filter_map(|part| match part {
+                                        ResponseContentPart::InputText { text } => {
+                                            Some(text.as_str())
+                                        }
+                                        _ => None,
+                                    })
+                                    .collect::<Vec<_>>()
+                                    .join(" ")
+                            }
+                        };
+
+                        messages.push(role_to_chat_message(role.as_str(), text));
+                    }
+                    ResponseInputOutputItem::Message { role, content, .. } => {
+                        // Extract text from content parts
+                        let text = extract_text_from_content(content);
+
+                        messages.push(role_to_chat_message(role.as_str(), text));
+                    }
+                    ResponseInputOutputItem::FunctionToolCall {
+                        id,
+                        name,
+                        arguments,
+                        output,
+                        ..
+                    } => {
+                        // Tool call from history - add as assistant message with tool call
+                        // followed by tool response if output exists
+
+                        // Add assistant message with tool_calls (the LLM's decision)
+                        messages.push(ChatMessage::Assistant {
+                            content: None,
+                            name: None,
+                            tool_calls: Some(vec![ToolCall {
+                                id: id.clone(),
+                                tool_type: "function".to_string(),
+                                function: FunctionCallResponse {
+                                    name: name.clone(),
+                                    arguments: Some(arguments.clone()),
+                                },
+                            }]),
+                            reasoning_content: None,
+                        });
+
+                        // Add tool result message if output exists
+                        if let Some(output_text) = output {
+                            messages.push(ChatMessage::Tool {
+                                content: MessageContent::Text(output_text.clone()),
+                                tool_call_id: id.clone(),
+                            });
+                        }
+                    }
+                    ResponseInputOutputItem::Reasoning { content, .. } => {
+                        // Reasoning content - add as assistant message with reasoning_content
+                        let reasoning_text = content
+                            .iter()
+                            .map(|c| match c {
+                                ReasoningText { text } => text.as_str(),
+                            })
+                            .collect::<Vec<_>>()
+                            .join("\n");
+
+                        messages.push(ChatMessage::Assistant {
+                            content: None,
+                            name: None,
+                            tool_calls: None,
+                            reasoning_content: Some(reasoning_text),
+                        });
+                    }
+                    ResponseInputOutputItem::FunctionCallOutput {
+                        call_id, output, ..
+                    } => {
+                        // Function call output - add as tool message
+                        // Note: The function name is looked up from prev_outputs in Harmony path
+                        // For Chat path, we just use the call_id
+                        messages.push(ChatMessage::Tool {
+                            content: MessageContent::Text(output.clone()),
+                            tool_call_id: call_id.clone(),
+                        });
+                    }
+                }
+            }
+        }
+    }
+
+    // Ensure we have at least one message
+    if messages.is_empty() {
+        return Err("Request must contain at least one message".to_string());
+    }
+
+    // 3. Extract function tools from ResponseTools
+    // Only function tools are extracted here (include_mcp: false).
+    // MCP tools are merged later by the tool loop (see tool_loop.rs:prepare_chat_tools_and_choice)
+    // before the chat pipeline, where tool_choice constraints are applied to ALL tools combined.
+    let function_tools = extract_tools_from_response_tools(req.tools.as_deref(), false);
+    let tools = if function_tools.is_empty() {
+        None
+    } else {
+        Some(function_tools)
+    };
+
+    // 4. Build ChatCompletionRequest
+    let is_streaming = req.stream.unwrap_or(false);
+
+    Ok(ChatCompletionRequest {
+        messages,
+        model: if req.model.is_empty() {
+            UNKNOWN_MODEL_ID.to_string()
+        } else {
+            req.model.clone()
+        },
+        temperature: req.temperature,
+        max_completion_tokens: req.max_output_tokens,
+        stream: is_streaming,
+        stream_options: if is_streaming {
+            Some(StreamOptions {
+                include_usage: Some(true),
+            })
+        } else {
+            None
+        },
+        parallel_tool_calls: req.parallel_tool_calls,
+        top_logprobs: req.top_logprobs,
+        top_p: req.top_p,
+        skip_special_tokens: true,
+        tools,
+        tool_choice: req.tool_choice.clone(),
+        response_format: map_text_to_response_format(&req.text),
+        ..Default::default()
+    })
+}
+
+/// Extract text content from ResponseContentPart array
+fn extract_text_from_content(content: &[ResponseContentPart]) -> String {
+    content
+        .iter()
+        .filter_map(|part| match part {
+            ResponseContentPart::InputText { text } => Some(text.as_str()),
+            ResponseContentPart::OutputText { text, .. } => Some(text.as_str()),
+            _ => None,
+        })
+        .collect::<Vec<_>>()
+        .join("")
+}
+
+/// Convert role and text to ChatMessage
+fn role_to_chat_message(role: &str, text: String) -> ChatMessage {
+    match role {
+        "user" => ChatMessage::User {
+            content: MessageContent::Text(text),
+            name: None,
+        },
+        "assistant" => ChatMessage::Assistant {
+            content: Some(MessageContent::Text(text)),
+            name: None,
+            tool_calls: None,
+            reasoning_content: None,
+        },
+        "system" => ChatMessage::System {
+            content: MessageContent::Text(text),
+            name: None,
+        },
+        _ => {
+            // Unknown role, treat as user message
+            ChatMessage::User {
+                content: MessageContent::Text(text),
+                name: None,
+            }
+        }
+    }
+}
+
+/// Map TextConfig from Responses API to ResponseFormat for Chat API
+///
+/// Converts the structured output configuration from the Responses API format
+/// to the Chat API format for non-Harmony models.
+fn map_text_to_response_format(text: &Option<TextConfig>) -> Option<ResponseFormat> {
+    let text_config = text.as_ref()?;
+    let format = text_config.format.as_ref()?;
+
+    match format {
+        TextFormat::Text => Some(ResponseFormat::Text),
+        TextFormat::JsonObject => Some(ResponseFormat::JsonObject),
+        TextFormat::JsonSchema {
+            name,
+            schema,
+            description: _,
+            strict,
+        } => Some(ResponseFormat::JsonSchema {
+            json_schema: JsonSchemaFormat {
+                name: name.clone(),
+                schema: schema.clone(),
+                strict: *strict,
+            },
+        }),
+    }
+}
+
+/// Convert a ChatCompletionResponse to ResponsesResponse
+///
+/// # Conversion Logic
+/// - `id` → `response_id_override` if provided, otherwise `chat_resp.id`
+/// - `model` → `model` (pass through)
+/// - `choices[0].message` → `output` array (convert to ResponseOutputItem::Message)
+/// - `choices[0].finish_reason` → determines `status` (stop/length → Completed)
+/// - `created` timestamp → `created_at`
+pub(crate) fn chat_to_responses(
+    chat_resp: &ChatCompletionResponse,
+    original_req: &ResponsesRequest,
+    response_id_override: Option<String>,
+) -> Result<ResponsesResponse, String> {
+    // Extract the first choice (responses API doesn't support n>1)
+    let choice = chat_resp
+        .choices
+        .first()
+        .ok_or_else(|| "Chat response contains no choices".to_string())?;
+
+    // Convert assistant message to output items
+    let mut output: Vec<ResponseOutputItem> = Vec::new();
+
+    // Convert message content to output item
+    if let Some(content) = &choice.message.content {
+        if !content.is_empty() {
+            output.push(ResponseOutputItem::Message {
+                id: format!("msg_{}", chat_resp.id),
+                role: "assistant".to_string(),
+                content: vec![ResponseContentPart::OutputText {
+                    text: content.clone(),
+                    annotations: vec![],
+                    logprobs: choice.logprobs.clone(),
+                }],
+                status: "completed".to_string(),
+            });
+        }
+    }
+
+    // Convert reasoning content if present (O1-style models)
+    if let Some(reasoning) = &choice.message.reasoning_content {
+        if !reasoning.is_empty() {
+            output.push(ResponseOutputItem::Reasoning {
+                id: format!("reasoning_{}", chat_resp.id),
+                summary: vec![],
+                content: vec![ReasoningText {
+                    text: reasoning.clone(),
+                }],
+                status: Some("completed".to_string()),
+            });
+        }
+    }
+
+    // Convert tool calls if present
+    if let Some(tool_calls) = &choice.message.tool_calls {
+        for tool_call in tool_calls {
+            output.push(ResponseOutputItem::FunctionToolCall {
+                id: tool_call.id.clone(),
+                call_id: tool_call.id.clone(),
+                name: tool_call.function.name.clone(),
+                arguments: tool_call.function.arguments.clone().unwrap_or_default(),
+                output: None, // Tool hasn't been executed yet
+                status: "in_progress".to_string(),
+            });
+        }
+    }
+
+    // Determine response status based on finish_reason
+    let status = match choice.finish_reason.as_deref() {
+        Some("stop") | Some("length") => ResponseStatus::Completed,
+        Some("tool_calls") => ResponseStatus::InProgress, // Waiting for tool execution
+        Some("failed") | Some("error") => ResponseStatus::Failed,
+        _ => ResponseStatus::Completed, // Default to completed
+    };
+
+    // Convert usage from Usage to UsageInfo, then wrap in ResponsesUsage
+    let usage = chat_resp.usage.as_ref().map(|u| {
+        let usage_info = UsageInfo {
+            prompt_tokens: u.prompt_tokens,
+            completion_tokens: u.completion_tokens,
+            total_tokens: u.total_tokens,
+            reasoning_tokens: u
+                .completion_tokens_details
+                .as_ref()
+                .and_then(|d| d.reasoning_tokens),
+            prompt_tokens_details: None, // Chat response doesn't have this
+        };
+        ResponsesUsage::Classic(usage_info)
+    });
+
+    // Generate response
+    let response_id = response_id_override.unwrap_or_else(|| chat_resp.id.clone());
+    Ok(ResponsesResponse::builder(&response_id, &chat_resp.model)
+        .copy_from_request(original_req)
+        .created_at(chat_resp.created as i64)
+        .status(status)
+        .output(output)
+        .maybe_text(original_req.text.clone())
+        .maybe_usage(usage)
+        .build())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_text_input_conversion() {
+        let req = ResponsesRequest {
+            input: ResponseInput::Text("Hello, world!".to_string()),
+            instructions: Some("You are a helpful assistant.".to_string()),
+            model: "gpt-4".to_string(),
+            temperature: Some(0.7),
+            ..Default::default()
+        };
+
+        let chat_req = responses_to_chat(&req).unwrap();
+        assert_eq!(chat_req.messages.len(), 2); // system + user
+        assert_eq!(chat_req.model, "gpt-4");
+        assert_eq!(chat_req.temperature, Some(0.7));
+    }
+
+    #[test]
+    fn test_items_input_conversion() {
+        let req = ResponsesRequest {
+            input: ResponseInput::Items(vec![
+                ResponseInputOutputItem::Message {
+                    id: "msg_1".to_string(),
+                    role: "user".to_string(),
+                    content: vec![ResponseContentPart::InputText {
+                        text: "Hello!".to_string(),
+                    }],
+                    status: None,
+                },
+                ResponseInputOutputItem::Message {
+                    id: "msg_2".to_string(),
+                    role: "assistant".to_string(),
+                    content: vec![ResponseContentPart::OutputText {
+                        text: "Hi there!".to_string(),
+                        annotations: vec![],
+                        logprobs: None,
+                    }],
+                    status: None,
+                },
+            ]),
+            ..Default::default()
+        };
+
+        let chat_req = responses_to_chat(&req).unwrap();
+        assert_eq!(chat_req.messages.len(), 2); // user + assistant
+    }
+
+    #[test]
+    fn test_empty_input_error() {
+        let req = ResponsesRequest {
+            input: ResponseInput::Text("".to_string()),
+            ..Default::default()
+        };
+
+        // Empty text should still create a user message, so this should succeed
+        let result = responses_to_chat(&req);
+        assert!(result.is_ok());
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/responses/handlers.rs

@@ -0,0 +1,158 @@
+//! Handler functions for /v1/responses endpoints
+//!
+//! # Public API
+//!
+//! - `route_responses()` - POST /v1/responses (main entry point)
+//!
+//! # Architecture
+//!
+//! This module provides the entry point for the /v1/responses endpoint.
+//! It supports two execution modes:
+//!
+//! 1. **Synchronous** - Returns complete response immediately (non_streaming.rs)
+//! 2. **Streaming** - Returns SSE stream with real-time events (streaming.rs)
+//!
+//! Note: Background mode is no longer supported. Requests with background=true
+//! will be rejected with a 400 error.
+//!
+//! # Request Flow
+//!
+//! ```text
+//! route_responses()
+//!   ├─► route_responses_sync()  → non_streaming::route_responses_internal()
+//!   └─► route_responses_streaming()
+//!       ├─► streaming::execute_tool_loop_streaming() (MCP tools)
+//!       └─► streaming::convert_chat_stream_to_responses_stream() (no MCP)
+//! ```
+
+use std::sync::Arc;
+
+use axum::{
+    http,
+    response::{IntoResponse, Response},
+};
+use tracing::debug;
+use uuid::Uuid;
+
+use super::{common::load_conversation_history, conversions, non_streaming, streaming};
+use crate::{
+    protocols::responses::ResponsesRequest,
+    routers::{
+        error,
+        grpc::common::responses::{ensure_mcp_connection, ResponsesContext},
+    },
+};
+
+/// Main handler for POST /v1/responses
+///
+/// Validates request, determines execution mode (sync/streaming), and delegates
+pub(crate) async fn route_responses(
+    ctx: &ResponsesContext,
+    request: Arc<ResponsesRequest>,
+    headers: Option<http::HeaderMap>,
+    model_id: Option<String>,
+) -> Response {
+    // 1. Reject background mode (no longer supported)
+    let is_background = request.background.unwrap_or(false);
+    if is_background {
+        return error::bad_request(
+            "unsupported_parameter",
+            "Background mode is not supported. Please set 'background' to false or omit it.",
+        );
+    }
+
+    // 2. Route based on execution mode
+    let is_streaming = request.stream.unwrap_or(false);
+    if is_streaming {
+        route_responses_streaming(ctx, request, headers, model_id).await
+    } else {
+        // Generate response ID for synchronous execution
+        let response_id = Some(format!("resp_{}", Uuid::new_v4()));
+        route_responses_sync(ctx, request, headers, model_id, response_id).await
+    }
+}
+
+// ============================================================================
+// Synchronous Entry Point
+// ============================================================================
+
+/// Execute synchronous responses request
+async fn route_responses_sync(
+    ctx: &ResponsesContext,
+    request: Arc<ResponsesRequest>,
+    headers: Option<http::HeaderMap>,
+    model_id: Option<String>,
+    response_id: Option<String>,
+) -> Response {
+    match non_streaming::route_responses_internal(ctx, request, headers, model_id, response_id)
+        .await
+    {
+        Ok(responses_response) => axum::Json(responses_response).into_response(),
+        Err(response) => response, // Already a Response with proper status code
+    }
+}
+
+// ============================================================================
+// Streaming Entry Point
+// ============================================================================
+
+/// Execute streaming responses request
+async fn route_responses_streaming(
+    ctx: &ResponsesContext,
+    request: Arc<ResponsesRequest>,
+    headers: Option<http::HeaderMap>,
+    model_id: Option<String>,
+) -> Response {
+    // 1. Load conversation history
+    let modified_request = match load_conversation_history(ctx, &request).await {
+        Ok(req) => req,
+        Err(response) => return response, // Already a Response with proper status code
+    };
+
+    // 2. Check MCP connection and get whether MCP tools are present
+    let (has_mcp_tools, server_keys) =
+        match ensure_mcp_connection(&ctx.mcp_manager, request.tools.as_deref()).await {
+            Ok(result) => result,
+            Err(response) => return response,
+        };
+
+    // Set the server keys in the context
+    {
+        let mut servers = ctx.requested_servers.write().unwrap();
+        *servers = server_keys;
+    }
+
+    if has_mcp_tools {
+        debug!("MCP tools detected in streaming mode, using streaming tool loop");
+
+        return streaming::execute_tool_loop_streaming(
+            ctx,
+            modified_request,
+            &request,
+            headers,
+            model_id,
+        )
+        .await;
+    }
+
+    // 3. Convert ResponsesRequest → ChatCompletionRequest
+    let chat_request = match conversions::responses_to_chat(&modified_request) {
+        Ok(req) => Arc::new(req),
+        Err(e) => {
+            return error::bad_request(
+                "convert_request_failed",
+                format!("Failed to convert request: {}", e),
+            );
+        }
+    };
+
+    // 4. Execute chat pipeline and convert streaming format (no MCP tools)
+    streaming::convert_chat_stream_to_responses_stream(
+        ctx,
+        chat_request,
+        headers,
+        model_id,
+        &request,
+    )
+    .await
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/responses/mod.rs

@@ -0,0 +1,20 @@
+//! Regular gRPC Router `/v1/responses` endpoint implementation
+//!
+//! This module handles all responses-specific logic for the regular (non-Harmony) pipeline.
+//!
+//! ## Architecture
+//!
+//! - `handlers` - Entry points: route_responses (thin dispatcher)
+//! - `non_streaming` - Non-streaming execution with MCP tool loop
+//! - `streaming` - Streaming execution with MCP tool loop
+//! - `common` - Shared helpers: ToolLoopState, tool preparation, MCP metadata builders
+//! - `conversions` - Request/response conversion between Responses and Chat formats
+
+mod common;
+mod conversions;
+mod handlers;
+mod non_streaming;
+mod streaming;
+
+// Public exports
+pub(crate) use handlers::route_responses;

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/responses/non_streaming.rs

@@ -0,0 +1,431 @@
+//! Non-streaming execution for Regular Responses API
+//!
+//! This module handles non-streaming request execution:
+//! - `route_responses_internal` - Core execution orchestration
+//! - `execute_tool_loop` - MCP tool loop execution
+//! - `execute_without_mcp` - Simple pipeline execution without MCP
+
+use std::{sync::Arc, time::Instant};
+
+use axum::response::Response;
+use serde_json::json;
+use tracing::{debug, error, trace, warn};
+
+use super::{
+    common::{
+        build_mcp_list_tools_item, build_next_request, convert_mcp_tools_to_chat_tools,
+        extract_all_tool_calls_from_chat, load_conversation_history, prepare_chat_tools_and_choice,
+        ExtractedToolCall, ToolLoopState,
+    },
+    conversions,
+};
+use crate::{
+    observability::metrics::{metrics_labels, Metrics},
+    protocols::responses::{ResponseStatus, ResponsesRequest, ResponsesResponse},
+    routers::{
+        error,
+        grpc::common::responses::{
+            ensure_mcp_connection, persist_response_if_needed, ResponsesContext,
+        },
+        mcp_utils::{extract_server_label, DEFAULT_MAX_ITERATIONS},
+    },
+};
+
+/// Internal implementation for non-streaming responses
+///
+/// This is the core execution path that:
+/// 1. Loads conversation history / response chain
+/// 2. Checks for MCP tools
+/// 3. Executes with or without MCP tool loop
+/// 4. Persists to storage
+pub(super) async fn route_responses_internal(
+    ctx: &ResponsesContext,
+    request: Arc<ResponsesRequest>,
+    headers: Option<http::HeaderMap>,
+    model_id: Option<String>,
+    response_id: Option<String>,
+) -> Result<ResponsesResponse, Response> {
+    // 1. Load conversation history and build modified request
+    let modified_request = load_conversation_history(ctx, &request).await?;
+
+    // 2. Check MCP connection and get whether MCP tools are present
+    let (has_mcp_tools, server_keys) =
+        ensure_mcp_connection(&ctx.mcp_manager, request.tools.as_deref()).await?;
+
+    // Set the server keys in the context
+    {
+        let mut servers = ctx.requested_servers.write().unwrap();
+        *servers = server_keys;
+    }
+
+    let responses_response = if has_mcp_tools {
+        debug!("MCP tools detected, using tool loop");
+
+        // Execute with MCP tool loop
+        execute_tool_loop(
+            ctx,
+            modified_request,
+            &request,
+            headers,
+            model_id,
+            response_id.clone(),
+        )
+        .await?
+    } else {
+        // No MCP tools - execute without MCP (may have function tools or no tools)
+        execute_without_mcp(
+            ctx,
+            &modified_request,
+            &request,
+            headers,
+            model_id,
+            response_id.clone(),
+        )
+        .await?
+    };
+
+    // 5. Persist response to storage if store=true
+    persist_response_if_needed(
+        ctx.conversation_storage.clone(),
+        ctx.conversation_item_storage.clone(),
+        ctx.response_storage.clone(),
+        &responses_response,
+        &request,
+    )
+    .await;
+
+    Ok(responses_response)
+}
+
+/// Execute request without MCP tool loop (simple pipeline execution)
+pub(super) async fn execute_without_mcp(
+    ctx: &ResponsesContext,
+    modified_request: &ResponsesRequest,
+    original_request: &ResponsesRequest,
+    headers: Option<http::HeaderMap>,
+    model_id: Option<String>,
+    response_id: Option<String>,
+) -> Result<ResponsesResponse, Response> {
+    // Convert ResponsesRequest → ChatCompletionRequest
+    let chat_request = conversions::responses_to_chat(modified_request).map_err(|e| {
+        error!(
+            function = "execute_without_mcp",
+            error = %e,
+            "Failed to convert ResponsesRequest to ChatCompletionRequest"
+        );
+        error::bad_request(
+            "convert_request_failed",
+            format!("Failed to convert request: {}", e),
+        )
+    })?;
+
+    // Execute chat pipeline (errors already have proper HTTP status codes)
+    let chat_response = ctx
+        .pipeline
+        .execute_chat_for_responses(
+            Arc::new(chat_request),
+            headers,
+            model_id,
+            ctx.components.clone(),
+        )
+        .await?; // Preserve the Response error as-is
+
+    // Convert ChatCompletionResponse → ResponsesResponse
+    conversions::chat_to_responses(&chat_response, original_request, response_id).map_err(|e| {
+        error!(
+            function = "execute_without_mcp",
+            error = %e,
+            "Failed to convert ChatCompletionResponse to ResponsesResponse"
+        );
+        error::internal_error(
+            "convert_to_responses_format_failed",
+            format!("Failed to convert to responses format: {}", e),
+        )
+    })
+}
+
+/// Execute the MCP tool calling loop
+///
+/// This wraps pipeline.execute_chat_for_responses() in a loop that:
+/// 1. Executes the chat pipeline
+/// 2. Checks if response has tool calls
+/// 3. If yes, executes MCP tools and builds resume request
+/// 4. Repeats until no more tool calls or limit reached
+pub(super) async fn execute_tool_loop(
+    ctx: &ResponsesContext,
+    mut current_request: ResponsesRequest,
+    original_request: &ResponsesRequest,
+    headers: Option<http::HeaderMap>,
+    model_id: Option<String>,
+    response_id: Option<String>,
+) -> Result<ResponsesResponse, Response> {
+    // Get server label from original request tools
+    let server_label = extract_server_label(original_request.tools.as_deref(), "request-mcp");
+
+    let mut state = ToolLoopState::new(original_request.input.clone(), server_label.clone());
+
+    // Configuration: max iterations as safety limit
+    let max_tool_calls = original_request.max_tool_calls.map(|n| n as usize);
+
+    trace!(
+        "Starting MCP tool loop: server_label={}, max_tool_calls={:?}, max_iterations={}",
+        server_label,
+        max_tool_calls,
+        DEFAULT_MAX_ITERATIONS
+    );
+
+    // Get MCP tools and convert to chat format (do this once before loop)
+    let mcp_tools = {
+        let servers = ctx.requested_servers.read().unwrap();
+        ctx.mcp_manager.list_tools_for_servers(&servers)
+    };
+    let mcp_chat_tools = convert_mcp_tools_to_chat_tools(&mcp_tools);
+    trace!(
+        "Converted {} MCP tools to chat format",
+        mcp_chat_tools.len()
+    );
+
+    loop {
+        // Convert to chat request
+        let mut chat_request = conversions::responses_to_chat(&current_request).map_err(|e| {
+            error!(
+                function = "tool_loop",
+                iteration = state.iteration,
+                error = %e,
+                "Failed to convert ResponsesRequest to ChatCompletionRequest in tool loop"
+            );
+            error::bad_request(
+                "convert_request_failed",
+                format!("Failed to convert request: {}", e),
+            )
+        })?;
+
+        // Prepare tools and tool_choice for this iteration
+        prepare_chat_tools_and_choice(&mut chat_request, &mcp_chat_tools, state.iteration);
+
+        // Execute chat pipeline (errors already have proper HTTP status codes)
+        let chat_response = ctx
+            .pipeline
+            .execute_chat_for_responses(
+                Arc::new(chat_request),
+                headers.clone(),
+                model_id.clone(),
+                ctx.components.clone(),
+            )
+            .await?;
+
+        // Check for function calls (extract all for parallel execution)
+        let tool_calls = extract_all_tool_calls_from_chat(&chat_response);
+
+        if !tool_calls.is_empty() {
+            state.iteration += 1;
+
+            // Record tool loop iteration metric
+            Metrics::record_mcp_tool_iteration(&current_request.model);
+
+            trace!(
+                "Tool loop iteration {}: found {} tool call(s)",
+                state.iteration,
+                tool_calls.len()
+            );
+
+            // Separate MCP and function tool calls
+            let mcp_tool_names: std::collections::HashSet<&str> =
+                mcp_tools.iter().map(|t| t.name.as_ref()).collect();
+            let (mcp_tool_calls, function_tool_calls): (Vec<ExtractedToolCall>, Vec<_>) =
+                tool_calls
+                    .into_iter()
+                    .partition(|tc| mcp_tool_names.contains(tc.name.as_str()));
+
+            trace!(
+                "Separated tool calls: {} MCP, {} function",
+                mcp_tool_calls.len(),
+                function_tool_calls.len()
+            );
+
+            // If ANY tool call is a function tool, return to caller immediately
+            if !function_tool_calls.is_empty() {
+                // Convert chat response to responses format (includes all tool calls)
+                let responses_response = conversions::chat_to_responses(
+                    &chat_response,
+                    original_request,
+                    response_id.clone(),
+                )
+                .map_err(|e| {
+                    error!(
+                        function = "tool_loop",
+                        iteration = state.iteration,
+                        error = %e,
+                        context = "function_tool_calls",
+                        "Failed to convert ChatCompletionResponse to ResponsesResponse"
+                    );
+                    error::internal_error(
+                        "convert_to_responses_format_failed",
+                        format!("Failed to convert to responses format: {}", e),
+                    )
+                })?;
+
+                // Return response with function tool calls to caller
+                return Ok(responses_response);
+            }
+
+            // All MCP tools - check combined limit BEFORE executing
+            let effective_limit = match max_tool_calls {
+                Some(user_max) => user_max.min(DEFAULT_MAX_ITERATIONS),
+                None => DEFAULT_MAX_ITERATIONS,
+            };
+
+            if state.total_calls + mcp_tool_calls.len() > effective_limit {
+                warn!(
+                    "Reached tool call limit: {} + {} > {} (max_tool_calls={:?}, safety_limit={})",
+                    state.total_calls,
+                    mcp_tool_calls.len(),
+                    effective_limit,
+                    max_tool_calls,
+                    DEFAULT_MAX_ITERATIONS
+                );
+
+                // Convert chat response to responses format and mark as incomplete
+                let mut responses_response = conversions::chat_to_responses(
+                    &chat_response,
+                    original_request,
+                    response_id.clone(),
+                )
+                .map_err(|e| {
+                    error!(
+                        function = "tool_loop",
+                        iteration = state.iteration,
+                        error = %e,
+                        context = "max_tool_calls_limit",
+                        "Failed to convert ChatCompletionResponse to ResponsesResponse"
+                    );
+                    error::internal_error(
+                        "convert_to_responses_format_failed",
+                        format!("Failed to convert to responses format: {}", e),
+                    )
+                })?;
+
+                // Mark as completed but with incomplete details
+                responses_response.status = ResponseStatus::Completed;
+                responses_response.incomplete_details = Some(json!({ "reason": "max_tool_calls" }));
+
+                return Ok(responses_response);
+            }
+
+            // Execute all MCP tools
+            for tool_call in mcp_tool_calls {
+                trace!(
+                    "Calling MCP tool '{}' (call_id: {}) with args: {}",
+                    tool_call.name,
+                    tool_call.call_id,
+                    tool_call.arguments
+                );
+
+                let tool_start = Instant::now();
+                let (output_str, success, error) = match ctx
+                    .mcp_manager
+                    .call_tool(tool_call.name.as_str(), tool_call.arguments.as_str())
+                    .await
+                {
+                    Ok(result) => match serde_json::to_string(&result) {
+                        Ok(output) => (output, true, None),
+                        Err(e) => {
+                            let err = format!("Failed to serialize tool result: {}", e);
+                            warn!("{}", err);
+                            let error_json = json!({ "error": &err }).to_string();
+                            (error_json, false, Some(err))
+                        }
+                    },
+                    Err(err) => {
+                        let err_str = format!("tool call failed: {}", err);
+                        warn!("Tool execution failed: {}", err_str);
+                        // Return error as output, let model decide how to proceed
+                        let error_json = json!({ "error": &err_str }).to_string();
+                        (error_json, false, Some(err_str))
+                    }
+                };
+                let tool_duration = tool_start.elapsed();
+
+                // Record MCP tool metrics
+                Metrics::record_mcp_tool_duration(
+                    &current_request.model,
+                    &tool_call.name,
+                    tool_duration,
+                );
+                Metrics::record_mcp_tool_call(
+                    &current_request.model,
+                    &tool_call.name,
+                    if success {
+                        metrics_labels::RESULT_SUCCESS
+                    } else {
+                        metrics_labels::RESULT_ERROR
+                    },
+                );
+
+                // Record the call in state
+                state.record_call(
+                    tool_call.call_id,
+                    tool_call.name,
+                    tool_call.arguments,
+                    output_str,
+                    success,
+                    error,
+                );
+
+                // Increment total calls counter
+                state.total_calls += 1;
+            }
+
+            // Build resume request with conversation history
+            current_request = build_next_request(&state, &current_request);
+
+            // Continue to next iteration
+        } else {
+            // No more tool calls, we're done
+            trace!(
+                "Tool loop completed: {} iterations, {} total calls",
+                state.iteration,
+                state.total_calls
+            );
+
+            // Convert final chat response to responses format
+            let mut responses_response = conversions::chat_to_responses(
+                &chat_response,
+                original_request,
+                response_id.clone(),
+            )
+            .map_err(|e| {
+                error!(
+                    function = "tool_loop",
+                    iteration = state.iteration,
+                    error = %e,
+                    context = "final_response",
+                    "Failed to convert ChatCompletionResponse to ResponsesResponse"
+                );
+                error::internal_error(
+                    "convert_to_responses_format_failed",
+                    format!("Failed to convert to responses format: {}", e),
+                )
+            })?;
+
+            // Inject MCP metadata into output
+            if state.total_calls > 0 {
+                // Prepend mcp_list_tools item
+                let servers = ctx.requested_servers.read().unwrap();
+                let mcp_list_tools =
+                    build_mcp_list_tools_item(&ctx.mcp_manager, &server_label, &servers);
+                responses_response.output.insert(0, mcp_list_tools);
+
+                // Append all mcp_call items at the end
+                responses_response.output.extend(state.mcp_call_items);
+
+                trace!(
+                    "Injected MCP metadata: 1 mcp_list_tools + {} mcp_call items",
+                    state.total_calls
+                );
+            }
+
+            return Ok(responses_response);
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/chat/mod.rs

@@ -0,0 +1,12 @@
+//! Chat endpoint pipeline stages
+//!
+//! These stages handle chat-specific preprocessing, request building, and response processing.
+//! They work with any model type by using injected model adapters.
+
+mod preparation;
+mod request_building;
+mod response_processing;
+
+pub(crate) use preparation::ChatPreparationStage;
+pub(crate) use request_building::ChatRequestBuildingStage;
+pub(crate) use response_processing::ChatResponseProcessingStage;

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/chat/preparation.rs

@@ -0,0 +1,119 @@
+//! Chat preparation stage: Filter tools, process messages, tokenize, build constraints
+
+use std::borrow::Cow;
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use crate::{
+    protocols::chat::ChatCompletionRequest,
+    routers::{
+        error,
+        grpc::{
+            common::stages::PipelineStage,
+            context::{PreparationOutput, RequestContext},
+            utils,
+        },
+    },
+};
+
+/// Chat preparation stage
+///
+/// Extracts chat-specific preparation logic from the old unified PreparationStage.
+/// This is a direct extraction without architectural changes.
+pub(crate) struct ChatPreparationStage;
+
+#[async_trait]
+impl PipelineStage for ChatPreparationStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let request = ctx.chat_request_arc();
+        self.prepare_chat(ctx, &request).await?;
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "ChatPreparation"
+    }
+}
+
+impl ChatPreparationStage {
+    async fn prepare_chat(
+        &self,
+        ctx: &mut RequestContext,
+        request: &ChatCompletionRequest,
+    ) -> Result<(), Response> {
+        // Step 0: Resolve tokenizer from registry (cached for reuse in response processing)
+        let tokenizer =
+            utils::resolve_tokenizer(ctx, "ChatPreparationStage::prepare_chat").map_err(|e| *e)?;
+
+        // Step 1: Filter tools if needed
+        let body_ref = utils::filter_chat_request_by_tool_choice(request);
+
+        // Step 2: Process messages and apply chat template
+        let processed_messages = match utils::process_chat_messages(&body_ref, &*tokenizer) {
+            Ok(msgs) => msgs,
+            Err(e) => {
+                error!(function = "ChatPreparationStage::execute", error = %e, "Failed to process chat messages");
+                return Err(error::bad_request("process_messages_failed", e));
+            }
+        };
+
+        // Step 3: Tokenize the processed text (no special tokens - chat template already handles them)
+        let encoding = match tokenizer.encode(&processed_messages.text, false) {
+            Ok(encoding) => encoding,
+            Err(e) => {
+                error!(function = "ChatPreparationStage::execute", error = %e, "Tokenization failed");
+                return Err(error::internal_error(
+                    "tokenization_failed",
+                    format!("Tokenization failed: {}", e),
+                ));
+            }
+        };
+
+        let token_ids = encoding.token_ids().to_vec();
+
+        // Step 4: Build tool constraints if needed
+        let tool_call_constraint = if let Some(tools) = body_ref.tools.as_ref() {
+            utils::generate_tool_constraints(tools, &request.tool_choice, &request.model)
+                .map_err(|e| {
+                    error!(function = "ChatPreparationStage::execute", error = %e, "Invalid tool configuration");
+                    error::bad_request("invalid_tool_configuration", format!("Invalid tool configuration: {}", e))
+                })?
+        } else {
+            None
+        };
+
+        // Step 5: Create stop sequence decoder (build once, reuse in non-stream)
+        let stop_decoder = utils::create_stop_decoder(
+            &tokenizer,
+            request.stop.as_ref(),
+            request.stop_token_ids.as_ref(),
+            request.skip_special_tokens,
+            request.no_stop_trim,
+        );
+
+        // Store results in context
+        ctx.state.preparation = Some(PreparationOutput {
+            original_text: Some(processed_messages.text.clone()),
+            token_ids,
+            processed_messages: Some(processed_messages),
+            tool_constraints: tool_call_constraint,
+            filtered_request: if matches!(body_ref, Cow::Owned(_)) {
+                Some(body_ref.into_owned())
+            } else {
+                None
+            },
+            // Harmony fields (not used for regular preparation)
+            harmony_mode: false,
+            selection_text: None,
+            harmony_messages: None,
+            harmony_stop_ids: None,
+        });
+
+        // Store stop decoder for reuse in response processing
+        ctx.state.response.stop_decoder = Some(stop_decoder);
+
+        Ok(())
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/chat/request_building.rs

@@ -0,0 +1,120 @@
+//! Chat request building stage: Build proto GenerateRequest for chat requests
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+use uuid::Uuid;
+
+use crate::routers::{
+    error,
+    grpc::{
+        client::GrpcClient,
+        common::stages::{helpers, PipelineStage},
+        context::{ClientSelection, RequestContext, WorkerSelection},
+        proto_wrapper::ProtoGenerateRequest,
+    },
+};
+
+/// Chat request building stage
+///
+/// Extracts chat-specific request building logic from the old unified RequestBuildingStage.
+pub(crate) struct ChatRequestBuildingStage {
+    inject_pd_metadata: bool,
+}
+
+impl ChatRequestBuildingStage {
+    pub fn new(inject_pd_metadata: bool) -> Self {
+        Self { inject_pd_metadata }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for ChatRequestBuildingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let prep = ctx.state.preparation.as_ref().ok_or_else(|| {
+            error!(
+                function = "ChatRequestBuildingStage::execute",
+                "Preparation not completed"
+            );
+            error::internal_error("preparation_not_completed", "Preparation not completed")
+        })?;
+
+        let clients = ctx.state.clients.as_ref().ok_or_else(|| {
+            error!(
+                function = "ChatRequestBuildingStage::execute",
+                "Client acquisition not completed"
+            );
+            error::internal_error(
+                "client_acquisition_not_completed",
+                "Client acquisition not completed",
+            )
+        })?;
+
+        let chat_request = ctx.chat_request_arc();
+
+        // Get client for building request (use prefill client if PD mode)
+        let builder_client = match clients {
+            ClientSelection::Single { client } => client,
+            ClientSelection::Dual { prefill, .. } => prefill,
+        };
+
+        // Build chat request
+        let request_id = format!("chatcmpl-{}", Uuid::new_v4());
+        let body_ref = prep.filtered_request.as_ref().unwrap_or(&chat_request);
+
+        // Dispatch to the appropriate client based on backend type
+        let mut proto_request = match builder_client {
+            GrpcClient::Sglang(sglang_client) => {
+                let req = sglang_client
+                    .build_generate_request_from_chat(
+                        request_id,
+                        body_ref,
+                        prep.processed_messages.as_ref().unwrap().text.clone(),
+                        prep.token_ids.clone(),
+                        prep.processed_messages
+                            .as_ref()
+                            .unwrap()
+                            .multimodal_inputs
+                            .clone(),
+                        prep.tool_constraints.clone(),
+                    )
+                    .map_err(|e| {
+                        error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build SGLang generate request");
+                        error::bad_request("invalid_request_parameters", format!("Invalid request parameters: {}", e))
+                    })?;
+                ProtoGenerateRequest::Sglang(Box::new(req))
+            }
+            GrpcClient::Vllm(vllm_client) => {
+                let req = vllm_client
+                    .build_generate_request_from_chat(
+                        request_id,
+                        body_ref,
+                        prep.processed_messages.as_ref().unwrap().text.clone(),
+                        prep.token_ids.clone(),
+                        prep.tool_constraints.clone(),
+                    )
+                    .map_err(|e| {
+                        error!(function = "ChatRequestBuildingStage::execute", error = %e, "Failed to build vLLM generate request");
+                        error::bad_request("invalid_request_parameters", format!("Invalid request parameters: {}", e))
+                    })?;
+                ProtoGenerateRequest::Vllm(Box::new(req))
+            }
+        };
+
+        // Inject PD metadata if needed
+        if self.inject_pd_metadata {
+            if let WorkerSelection::Dual { prefill, .. } = ctx.state.workers.as_ref().unwrap() {
+                helpers::inject_bootstrap_metadata(&mut proto_request, prefill);
+            }
+        }
+
+        ctx.state.proto_request = Some(
+            crate::routers::grpc::proto_wrapper::ProtoRequest::Generate(proto_request),
+        );
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "ChatRequestBuilding"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/chat/response_processing.rs

@@ -0,0 +1,146 @@
+//! Chat response processing stage: Handles both streaming and non-streaming responses
+//!
+//! - For streaming: Spawns background task and returns SSE response (early exit)
+//! - For non-streaming: Collects all responses and builds final ChatCompletionResponse
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use crate::{
+    core::AttachedBody,
+    routers::{
+        error,
+        grpc::{
+            common::stages::PipelineStage,
+            context::{FinalResponse, RequestContext},
+            regular::{processor, streaming},
+        },
+    },
+};
+
+/// Chat response processing stage
+pub(crate) struct ChatResponseProcessingStage {
+    processor: processor::ResponseProcessor,
+    streaming_processor: Arc<streaming::StreamingProcessor>,
+}
+
+impl ChatResponseProcessingStage {
+    pub fn new(
+        processor: processor::ResponseProcessor,
+        streaming_processor: Arc<streaming::StreamingProcessor>,
+    ) -> Self {
+        Self {
+            processor,
+            streaming_processor,
+        }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for ChatResponseProcessingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        self.process_chat_response(ctx).await
+    }
+
+    fn name(&self) -> &'static str {
+        "ChatResponseProcessing"
+    }
+}
+
+impl ChatResponseProcessingStage {
+    async fn process_chat_response(
+        &self,
+        ctx: &mut RequestContext,
+    ) -> Result<Option<Response>, Response> {
+        let is_streaming = ctx.is_streaming();
+
+        // Extract execution result
+        let execution_result = ctx.state.response.execution_result.take().ok_or_else(|| {
+            error!(
+                function = "ChatResponseProcessingStage::execute",
+                "No execution result"
+            );
+            error::internal_error("no_execution_result", "No execution result")
+        })?;
+
+        // Get dispatch metadata (needed by both streaming and non-streaming)
+        let dispatch = ctx
+            .state
+            .dispatch
+            .as_ref()
+            .ok_or_else(|| {
+                error!(
+                    function = "ChatResponseProcessingStage::execute",
+                    "Dispatch metadata not set"
+                );
+                error::internal_error("dispatch_metadata_not_set", "Dispatch metadata not set")
+            })?
+            .clone();
+
+        // Get cached tokenizer (resolved once in preparation stage)
+        let tokenizer = ctx.tokenizer_arc().ok_or_else(|| {
+            error!(
+                function = "ChatResponseProcessingStage::process_chat_response",
+                "Tokenizer not cached in context"
+            );
+            error::internal_error(
+                "tokenizer_not_cached",
+                "Tokenizer not cached in context - preparation stage may have been skipped",
+            )
+        })?;
+
+        if is_streaming {
+            // Streaming: Use StreamingProcessor and return SSE response
+            let response = self.streaming_processor.clone().process_streaming_response(
+                execution_result,
+                ctx.chat_request_arc(), // Cheap Arc clone (8 bytes)
+                dispatch,
+                tokenizer,
+            );
+
+            // Attach load guards to response body for proper RAII lifecycle
+            let response = match ctx.state.load_guards.take() {
+                Some(guards) => AttachedBody::wrap_response(response, guards),
+                None => response,
+            };
+
+            return Ok(Some(response));
+        }
+
+        // Non-streaming: Delegate to ResponseProcessor
+        let request_logprobs = ctx.chat_request().logprobs;
+
+        let chat_request = ctx.chat_request_arc();
+
+        let stop_decoder = ctx.state.response.stop_decoder.as_mut().ok_or_else(|| {
+            error!(
+                function = "ChatResponseProcessingStage::execute",
+                "Stop decoder not initialized"
+            );
+            error::internal_error(
+                "stop_decoder_not_initialized",
+                "Stop decoder not initialized",
+            )
+        })?;
+
+        let response = self
+            .processor
+            .process_non_streaming_chat_response(
+                execution_result,
+                chat_request,
+                dispatch,
+                tokenizer,
+                stop_decoder,
+                request_logprobs,
+            )
+            .await?;
+
+        // Store the final response
+        ctx.state.response.final_response = Some(FinalResponse::Chat(response));
+
+        Ok(None)
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/classify/mod.rs

@@ -0,0 +1,9 @@
+//! Pipeline stages for classify requests.
+//!
+//! Classify reuses embedding stages for preparation and request building,
+//! as the scheduler treats classify as an embedding request and returns logits.
+//! Only response processing is classify-specific (softmax + label mapping).
+
+pub(crate) mod response_processing;
+
+pub(crate) use response_processing::ClassifyResponseProcessingStage;

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/classify/response_processing.rs

@@ -0,0 +1,300 @@
+//! Response processing stage for classify requests.
+//!
+//! Key responsibilities:
+//! 1. Extract embedding (logits) from EmbedComplete response
+//! 2. Apply softmax to convert logits to probabilities
+//! 3. Find predicted class (argmax)
+//! 4. Map class index to label (from id2label or generic LABEL_N)
+//! 5. Build ClassifyResponse
+
+use std::collections::HashMap;
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use crate::{
+    protocols::{
+        classify::{ClassifyData, ClassifyResponse},
+        common::UsageInfo,
+    },
+    routers::{
+        error,
+        grpc::{
+            common::stages::PipelineStage,
+            context::{ExecutionResult, FinalResponse, RequestContext, WorkerSelection},
+        },
+    },
+};
+
+/// Response processing stage for classify requests.
+///
+/// Takes the logits from the embedding response and converts them to
+/// classification results with probabilities and labels.
+///
+/// The stage is stateless - id2label mapping is obtained from the
+/// selected worker's model card at runtime.
+pub(crate) struct ClassifyResponseProcessingStage;
+
+impl ClassifyResponseProcessingStage {
+    /// Create a new classify response processing stage.
+    pub fn new() -> Self {
+        Self
+    }
+
+    /// Apply softmax to logits to get probability distribution.
+    ///
+    /// Uses the numerically stable formula: softmax(x)_i = exp(x_i - max(x)) / sum(exp(x - max(x)))
+    fn softmax(logits: &[f32]) -> Vec<f32> {
+        if logits.is_empty() {
+            return vec![];
+        }
+
+        // Find max for numerical stability
+        let max_logit = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
+
+        // Compute exp(x - max) for each element
+        let exp_vals: Vec<f32> = logits.iter().map(|&x| (x - max_logit).exp()).collect();
+
+        // Sum of exponentials
+        let sum: f32 = exp_vals.iter().sum();
+
+        // Normalize to get probabilities
+        if sum == 0.0 {
+            // Avoid division by zero - return uniform distribution
+            let n = exp_vals.len();
+            return vec![1.0 / n as f32; n];
+        }
+
+        exp_vals.iter().map(|&x| x / sum).collect()
+    }
+
+    /// Find the index of the maximum value (argmax).
+    fn argmax(probs: &[f32]) -> u32 {
+        probs
+            .iter()
+            .enumerate()
+            .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
+            .map(|(idx, _)| idx as u32)
+            .unwrap_or(0)
+    }
+
+    /// Get label for a class index.
+    ///
+    /// Returns the label from id2label if available, otherwise returns generic "LABEL_N".
+    fn get_label(id2label: &HashMap<u32, String>, class_idx: u32) -> String {
+        id2label
+            .get(&class_idx)
+            .cloned()
+            .unwrap_or_else(|| format!("LABEL_{}", class_idx))
+    }
+
+    /// Extract id2label mapping from the selected worker's model card.
+    fn get_id2label_from_context(ctx: &RequestContext) -> HashMap<u32, String> {
+        // Get the selected worker
+        let worker = match ctx.state.workers.as_ref() {
+            Some(WorkerSelection::Single { worker }) => worker,
+            Some(WorkerSelection::Dual { prefill, .. }) => prefill, // Use prefill worker for model info
+            None => return HashMap::new(),
+        };
+
+        // Get id2label from the first model card
+        worker
+            .metadata()
+            .models
+            .first()
+            .map(|model| model.id2label.clone())
+            .unwrap_or_default()
+    }
+}
+
+impl Default for ClassifyResponseProcessingStage {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl PipelineStage for ClassifyResponseProcessingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        // Extract execution result
+        let execution_result = ctx.state.response.execution_result.take().ok_or_else(|| {
+            error!(
+                function = "ClassifyResponseProcessingStage::execute",
+                "Execution result missing"
+            );
+            error::internal_error("execution_result_missing", "Execution result missing")
+        })?;
+
+        // Expect Embedding result variant (classify uses embed backend)
+        let proto_response = if let ExecutionResult::Embedding { response } = execution_result {
+            response
+        } else {
+            error!(
+                function = "ClassifyResponseProcessingStage::execute",
+                "Invalid execution result: expected Embedding"
+            );
+            return Err(error::internal_error(
+                "invalid_execution_result",
+                "Expected Embedding result for classify",
+            ));
+        };
+
+        // Get logits from embedding response
+        let logits = proto_response.embedding();
+
+        if logits.is_empty() {
+            error!(
+                function = "ClassifyResponseProcessingStage::execute",
+                "Empty logits received from scheduler"
+            );
+            return Err(error::internal_error(
+                "empty_logits",
+                "Empty logits received from scheduler",
+            ));
+        }
+
+        // Get id2label from the worker's model card
+        let id2label = Self::get_id2label_from_context(ctx);
+
+        // Apply softmax to get probabilities
+        let probs = Self::softmax(logits);
+
+        // Get predicted class (argmax)
+        let predicted_class = Self::argmax(&probs);
+
+        // Get label for predicted class
+        let label = Self::get_label(&id2label, predicted_class);
+
+        // Build classify data
+        let classify_data = ClassifyData {
+            index: 0,
+            label,
+            probs: probs.clone(),
+            num_classes: probs.len() as u32,
+        };
+
+        // Get dispatch metadata
+        let dispatch = ctx.state.dispatch.as_ref().ok_or_else(|| {
+            error!(
+                function = "ClassifyResponseProcessingStage::execute",
+                "Dispatch metadata missing"
+            );
+            error::internal_error("dispatch_missing", "Dispatch metadata missing")
+        })?;
+
+        // Build usage info
+        let prompt_tokens = proto_response.prompt_tokens().max(0) as u32;
+        let usage = UsageInfo {
+            prompt_tokens,
+            total_tokens: prompt_tokens,
+            completion_tokens: 0,
+            prompt_tokens_details: None,
+            reasoning_tokens: None,
+        };
+
+        // Build response
+        let response = ClassifyResponse::new(
+            dispatch.request_id.clone(),
+            dispatch.model.clone(),
+            dispatch.created,
+            vec![classify_data],
+            usage,
+        );
+
+        // Store in context for pipeline to extract
+        ctx.state.response.final_response = Some(FinalResponse::Classify(response));
+
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "ClassifyResponseProcessing"
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_softmax_basic() {
+        let logits = vec![1.0, 2.0, 3.0];
+        let probs = ClassifyResponseProcessingStage::softmax(&logits);
+
+        // Probabilities should sum to 1
+        let sum: f32 = probs.iter().sum();
+        assert!((sum - 1.0).abs() < 1e-6);
+
+        // Highest logit should have highest probability
+        assert!(probs[2] > probs[1]);
+        assert!(probs[1] > probs[0]);
+    }
+
+    #[test]
+    fn test_softmax_empty() {
+        let probs = ClassifyResponseProcessingStage::softmax(&[]);
+        assert!(probs.is_empty());
+    }
+
+    #[test]
+    fn test_softmax_single() {
+        let probs = ClassifyResponseProcessingStage::softmax(&[5.0]);
+        assert_eq!(probs.len(), 1);
+        assert!((probs[0] - 1.0).abs() < 1e-6);
+    }
+
+    #[test]
+    fn test_softmax_numerical_stability() {
+        // Large values that would overflow without max subtraction
+        let logits = vec![1000.0, 1001.0, 1002.0];
+        let probs = ClassifyResponseProcessingStage::softmax(&logits);
+
+        let sum: f32 = probs.iter().sum();
+        assert!((sum - 1.0).abs() < 1e-6);
+        assert!(probs[2] > probs[1]);
+    }
+
+    #[test]
+    fn test_argmax() {
+        assert_eq!(ClassifyResponseProcessingStage::argmax(&[0.1, 0.7, 0.2]), 1);
+        assert_eq!(
+            ClassifyResponseProcessingStage::argmax(&[0.9, 0.05, 0.05]),
+            0
+        );
+        assert_eq!(ClassifyResponseProcessingStage::argmax(&[0.1, 0.1, 0.8]), 2);
+    }
+
+    #[test]
+    fn test_get_label_with_mapping() {
+        let mut id2label = HashMap::new();
+        id2label.insert(0, "negative".to_string());
+        id2label.insert(1, "positive".to_string());
+
+        assert_eq!(
+            ClassifyResponseProcessingStage::get_label(&id2label, 0),
+            "negative"
+        );
+        assert_eq!(
+            ClassifyResponseProcessingStage::get_label(&id2label, 1),
+            "positive"
+        );
+        assert_eq!(
+            ClassifyResponseProcessingStage::get_label(&id2label, 2),
+            "LABEL_2"
+        ); // Fallback for unknown
+    }
+
+    #[test]
+    fn test_get_label_without_mapping() {
+        let id2label = HashMap::new();
+        assert_eq!(
+            ClassifyResponseProcessingStage::get_label(&id2label, 0),
+            "LABEL_0"
+        );
+        assert_eq!(
+            ClassifyResponseProcessingStage::get_label(&id2label, 5),
+            "LABEL_5"
+        );
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/embedding/mod.rs

@@ -0,0 +1,3 @@
+pub(crate) mod preparation;
+pub(crate) mod request_building;
+pub(crate) mod response_processing;

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/embedding/preparation.rs

@@ -0,0 +1,96 @@
+//! Preparation stage for embedding requests
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use crate::{
+    protocols::common::GenerationRequest,
+    routers::{
+        error,
+        grpc::{
+            common::stages::PipelineStage,
+            context::{PreparationOutput, RequestContext, RequestType},
+            utils,
+        },
+    },
+};
+
+pub(crate) struct EmbeddingPreparationStage;
+
+impl EmbeddingPreparationStage {
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for EmbeddingPreparationStage {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl PipelineStage for EmbeddingPreparationStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        // Extract text from embedding or classify request (both use same preparation)
+        let text = match &ctx.input.request_type {
+            RequestType::Embedding(req) => req.extract_text_for_routing(),
+            RequestType::Classify(req) => req.extract_text_for_routing(),
+            _ => {
+                error!(
+                    function = "EmbeddingPreparationStage::execute",
+                    "Invalid request type: expected Embedding or Classify"
+                );
+                return Err(error::internal_error(
+                    "invalid_request_type",
+                    "Expected Embedding or Classify request",
+                ));
+            }
+        };
+        if text.is_empty() {
+            return Err(error::bad_request(
+                "empty_input",
+                "Input text cannot be empty",
+            ));
+        }
+
+        // Resolve tokenizer from registry (cached for potential reuse)
+        let tokenizer =
+            utils::resolve_tokenizer(ctx, "EmbeddingPreparationStage::execute").map_err(|e| *e)?;
+
+        // Tokenize with special tokens (BOS/EOS) for embeddings
+        // This matches Python's transformers behavior which reads add_bos_token/add_eos_token from tokenizer_config.json
+        let token_ids = tokenizer
+            .encode(&text, true)
+            .map_err(|e| {
+                error!(
+                    function = "EmbeddingPreparationStage::execute",
+                    error = %e,
+                    "Tokenization failed"
+                );
+                error::bad_request("tokenization_failed", format!("Tokenization failed: {}", e))
+            })?
+            .token_ids()
+            .to_vec();
+
+        // Store preparation output
+        ctx.state.preparation = Some(PreparationOutput {
+            original_text: Some(text),
+            token_ids,
+            processed_messages: None,
+            tool_constraints: None,
+            filtered_request: None,
+            harmony_mode: false,
+            selection_text: None,
+            harmony_messages: None,
+            harmony_stop_ids: None,
+        });
+
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "EmbeddingPreparation"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/embedding/request_building.rs

@@ -0,0 +1,104 @@
+//! Request building stage for embedding requests
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+use uuid::Uuid;
+
+use crate::routers::{
+    error,
+    grpc::{
+        common::stages::PipelineStage,
+        context::{RequestContext, RequestType},
+        proto_wrapper::{ProtoEmbedRequest, ProtoRequest},
+    },
+};
+
+/// Request building stage for embedding requests
+pub(crate) struct EmbeddingRequestBuildingStage;
+
+impl EmbeddingRequestBuildingStage {
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for EmbeddingRequestBuildingStage {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl PipelineStage for EmbeddingRequestBuildingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        // Extract log_metrics from embedding or classify request (both use same backend)
+        let log_metrics = match &ctx.input.request_type {
+            RequestType::Embedding(req) => req.log_metrics,
+            RequestType::Classify(req) => req.log_metrics,
+            _ => {
+                error!(
+                    function = "EmbeddingRequestBuildingStage::execute",
+                    "Invalid request type: expected Embedding or Classify"
+                );
+                return Err(error::internal_error(
+                    "invalid_request_type",
+                    "Expected Embedding or Classify request",
+                ));
+            }
+        };
+
+        // Preparation output should have tokenized input
+        let prep_output = ctx.state.preparation.as_ref().ok_or_else(|| {
+            error!(
+                function = "EmbeddingRequestBuildingStage::execute",
+                "Preparation output missing"
+            );
+            error::internal_error("preparation_missing", "Preparation output missing")
+        })?;
+
+        // Extract client
+        let client = ctx
+            .state
+            .clients
+            .as_ref()
+            .and_then(|c| c.single())
+            .ok_or_else(|| {
+                error!(
+                    function = "EmbeddingRequestBuildingStage::execute",
+                    "Client not selected"
+                );
+                error::internal_error("client_missing", "Client not selected")
+            })?;
+
+        // Generate request ID with appropriate prefix based on request type
+        let request_id = match &ctx.input.request_type {
+            RequestType::Embedding(_) => format!("embed-{}", Uuid::new_v4()),
+            RequestType::Classify(_) => format!("classify-{}", Uuid::new_v4()),
+            _ => format!("embed-{}", Uuid::new_v4()), // fallback
+        };
+
+        // Extract original text
+        let original_text = prep_output.original_text.clone();
+
+        // Use backend-specific builder to create ProtoEmbedRequest
+        // Currently only SGLang supports embedding via gRPC
+        let sglang_client = client.as_sglang();
+
+        let sglang_req = sglang_client.build_embed_request(
+            request_id.clone(),
+            original_text,
+            prep_output.token_ids.clone(),
+            log_metrics,
+        );
+
+        let proto_req = ProtoEmbedRequest::Sglang(Box::new(sglang_req));
+
+        ctx.state.proto_request = Some(ProtoRequest::Embed(proto_req));
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "EmbeddingRequestBuilding"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/embedding/response_processing.rs

@@ -0,0 +1,119 @@
+//! Response processing stage for embedding requests
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use crate::{
+    protocols::embedding::{EmbeddingObject, EmbeddingResponse},
+    routers::{
+        error,
+        grpc::{
+            common::stages::PipelineStage,
+            context::{ExecutionResult, FinalResponse, RequestContext},
+            proto_wrapper::ProtoEmbedComplete,
+        },
+    },
+};
+
+/// Response processing stage for embedding requests
+pub(crate) struct EmbeddingResponseProcessingStage;
+
+impl EmbeddingResponseProcessingStage {
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl Default for EmbeddingResponseProcessingStage {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl PipelineStage for EmbeddingResponseProcessingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        // Extract execution result
+        let execution_result = ctx.state.response.execution_result.take().ok_or_else(|| {
+            error!(
+                function = "EmbeddingResponseProcessingStage::execute",
+                "Execution result missing"
+            );
+            error::internal_error("execution_result_missing", "Execution result missing")
+        })?;
+
+        // Expect Embedding result variant
+        let proto_response = if let ExecutionResult::Embedding { response } = execution_result {
+            response
+        } else {
+            error!(
+                function = "EmbeddingResponseProcessingStage::execute",
+                "Invalid execution result: expected Embedding"
+            );
+            return Err(error::internal_error(
+                "invalid_execution_result",
+                "Expected Embedding result",
+            ));
+        };
+
+        // Convert proto response to HTTP response
+        let embedding_response = self
+            .convert_response(ctx, proto_response)
+            .map_err(|boxed_err| *boxed_err)?;
+
+        // Store in context for pipeline to extract
+        ctx.state.response.final_response = Some(FinalResponse::Embedding(embedding_response));
+
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "EmbeddingResponseProcessing"
+    }
+}
+
+impl EmbeddingResponseProcessingStage {
+    fn convert_response(
+        &self,
+        ctx: &RequestContext,
+        proto: ProtoEmbedComplete,
+    ) -> Result<EmbeddingResponse, Box<Response>> {
+        let dispatch = ctx.state.dispatch.as_ref().ok_or_else(|| {
+            error!(
+                function = "EmbeddingResponseProcessingStage::convert_response",
+                "Dispatch metadata missing in context"
+            );
+            error::internal_error("dispatch_missing", "Dispatch metadata missing")
+        })?;
+
+        let model = dispatch.model.clone();
+
+        // Convert flat embedding vector to response
+        // single input -> single embedding object
+
+        let embedding_data = EmbeddingObject {
+            object: "embedding".to_string(),
+            embedding: proto.embedding().to_vec(),
+            index: 0,
+        };
+
+        // Casting i32 to u32 for usage stats
+        let prompt_tokens = proto.prompt_tokens().max(0) as u32;
+
+        let usage = crate::protocols::common::UsageInfo {
+            prompt_tokens,
+            total_tokens: prompt_tokens, // Embedding has no completion tokens
+            completion_tokens: 0,
+            prompt_tokens_details: None,
+            reasoning_tokens: None,
+        };
+
+        Ok(EmbeddingResponse {
+            object: "list".to_string(),
+            data: vec![embedding_data],
+            model,
+            usage,
+        })
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/generate/mod.rs

@@ -0,0 +1,12 @@
+//! Generate endpoint pipeline stages
+//!
+//! These stages handle generate-specific preprocessing, request building, and response processing.
+//! They work with any model type by using injected model adapters.
+
+mod preparation;
+mod request_building;
+mod response_processing;
+
+pub(crate) use preparation::GeneratePreparationStage;
+pub(crate) use request_building::GenerateRequestBuildingStage;
+pub(crate) use response_processing::GenerateResponseProcessingStage;

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/generate/preparation.rs

@@ -0,0 +1,128 @@
+//! Generate preparation stage: Resolve input, tokenize, create stop decoder
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use crate::{
+    protocols::{common::InputIds, generate::GenerateRequest},
+    routers::{
+        error,
+        grpc::{
+            common::stages::PipelineStage,
+            context::{PreparationOutput, RequestContext},
+            utils,
+        },
+    },
+    tokenizer::traits::Tokenizer,
+};
+
+/// Generate preparation stage
+///
+/// Extracts generate-specific preparation logic from the old unified PreparationStage.
+/// This is a direct extraction without architectural changes.
+pub(crate) struct GeneratePreparationStage;
+
+#[async_trait]
+impl PipelineStage for GeneratePreparationStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let request = ctx.generate_request_arc();
+        self.prepare_generate(ctx, &request).await?;
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "GeneratePreparation"
+    }
+}
+
+impl GeneratePreparationStage {
+    async fn prepare_generate(
+        &self,
+        ctx: &mut RequestContext,
+        request: &GenerateRequest,
+    ) -> Result<(), Response> {
+        // Resolve tokenizer from registry (cached for reuse in response processing)
+        let tokenizer = utils::resolve_tokenizer(ctx, "GeneratePreparationStage::prepare_generate")
+            .map_err(|e| *e)?;
+
+        let (original_text, token_ids) = match self.resolve_generate_input(request, &tokenizer) {
+            Ok(res) => res,
+            Err(msg) => {
+                error!(function = "GeneratePreparationStage::execute", error = %msg, "Failed to resolve generate input");
+                return Err(error::bad_request("resolve_input_failed", msg));
+            }
+        };
+
+        // Create stop sequence decoder for generate requests
+        let params = request.sampling_params.as_ref();
+        let stop_decoder = utils::create_stop_decoder(
+            &tokenizer,
+            params.and_then(|p| p.stop.as_ref()),
+            params.and_then(|p| p.stop_token_ids.as_ref()),
+            params.and_then(|p| p.skip_special_tokens).unwrap_or(true),
+            params.and_then(|p| p.no_stop_trim).unwrap_or(false),
+        );
+
+        ctx.state.preparation = Some(PreparationOutput {
+            original_text,
+            token_ids,
+            processed_messages: None,
+            tool_constraints: None,
+            filtered_request: None,
+            // Harmony fields (not used for generate requests)
+            harmony_mode: false,
+            selection_text: None,
+            harmony_messages: None,
+            harmony_stop_ids: None,
+        });
+
+        // Store stop decoder
+        ctx.state.response.stop_decoder = Some(stop_decoder);
+
+        Ok(())
+    }
+
+    fn resolve_generate_input(
+        &self,
+        request: &GenerateRequest,
+        tokenizer: &Arc<dyn Tokenizer>,
+    ) -> Result<(Option<String>, Vec<u32>), String> {
+        if let Some(text) = &request.text {
+            return self
+                .tokenize_single_text(tokenizer, text)
+                .map(|(original, ids)| (Some(original), ids));
+        }
+
+        // Handle input_ids - validate and convert
+        if let Some(input_ids) = &request.input_ids {
+            return match input_ids {
+                InputIds::Single(ids) => ids
+                    .iter()
+                    .map(|&id| u32::try_from(id))
+                    .collect::<Result<Vec<u32>, _>>()
+                    .map(|converted| (None, converted))
+                    .map_err(|_| "input_ids must be non-negative".to_string()),
+                InputIds::Batch(_) => {
+                    Err("Batch input_ids are not supported over gRPC generate yet".to_string())
+                }
+            };
+        }
+
+        Err("Either `text` or `input_ids` must be provided".to_string())
+    }
+
+    fn tokenize_single_text(
+        &self,
+        tokenizer: &Arc<dyn Tokenizer>,
+        text: &str,
+    ) -> Result<(String, Vec<u32>), String> {
+        // Don't add special tokens - raw text generation uses text as-is
+        let encoding = tokenizer
+            .encode(text, false)
+            .map_err(|e| format!("Tokenization failed: {}", e))?;
+        Ok((text.to_string(), encoding.token_ids().to_vec()))
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/generate/request_building.rs

@@ -0,0 +1,115 @@
+//! Generate request building stage: Build proto GenerateRequest for generate requests
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+use uuid::Uuid;
+
+use crate::routers::{
+    error,
+    grpc::{
+        client::GrpcClient,
+        common::stages::{helpers, PipelineStage},
+        context::{ClientSelection, RequestContext, WorkerSelection},
+        proto_wrapper::ProtoGenerateRequest,
+    },
+};
+
+/// Generate request building stage
+///
+/// Extracts generate-specific request building logic from the old unified RequestBuildingStage.
+pub(crate) struct GenerateRequestBuildingStage {
+    inject_pd_metadata: bool,
+}
+
+impl GenerateRequestBuildingStage {
+    pub fn new(inject_pd_metadata: bool) -> Self {
+        Self { inject_pd_metadata }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for GenerateRequestBuildingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        let prep = ctx.state.preparation.as_ref().ok_or_else(|| {
+            error!(
+                function = "GenerateRequestBuildingStage::execute",
+                "Preparation not completed"
+            );
+            error::internal_error("preparation_not_completed", "Preparation not completed")
+        })?;
+
+        let clients = ctx.state.clients.as_ref().ok_or_else(|| {
+            error!(
+                function = "GenerateRequestBuildingStage::execute",
+                "Client acquisition not completed"
+            );
+            error::internal_error(
+                "client_acquisition_not_completed",
+                "Client acquisition not completed",
+            )
+        })?;
+
+        let generate_request = ctx.generate_request_arc();
+
+        // Get client for building request (use prefill client if PD mode)
+        let builder_client = match clients {
+            ClientSelection::Single { client } => client,
+            ClientSelection::Dual { prefill, .. } => prefill,
+        };
+
+        // Build generate request
+        let request_id = generate_request
+            .rid
+            .clone()
+            .unwrap_or_else(|| format!("gen-{}", Uuid::new_v4()));
+
+        // Dispatch to the appropriate client based on backend type
+        let mut proto_request = match builder_client {
+            GrpcClient::Sglang(sglang_client) => {
+                let req = sglang_client
+                    .build_plain_generate_request(
+                        request_id,
+                        &generate_request,
+                        prep.original_text.clone(),
+                        prep.token_ids.clone(),
+                    )
+                    .map_err(|e| {
+                        error!(function = "GenerateRequestBuildingStage::execute", error = %e, "Failed to build SGLang generate request");
+                        error::bad_request("build_request_failed", e)
+                    })?;
+                ProtoGenerateRequest::Sglang(Box::new(req))
+            }
+            GrpcClient::Vllm(vllm_client) => {
+                let req = vllm_client
+                    .build_plain_generate_request(
+                        request_id,
+                        &generate_request,
+                        prep.original_text.clone(),
+                        prep.token_ids.clone(),
+                    )
+                    .map_err(|e| {
+                        error!(function = "GenerateRequestBuildingStage::execute", error = %e, "Failed to build vLLM generate request");
+                        error::bad_request("build_request_failed", e)
+                    })?;
+                ProtoGenerateRequest::Vllm(Box::new(req))
+            }
+        };
+
+        // Inject PD metadata if needed
+        if self.inject_pd_metadata {
+            if let WorkerSelection::Dual { prefill, .. } = ctx.state.workers.as_ref().unwrap() {
+                helpers::inject_bootstrap_metadata(&mut proto_request, prefill);
+            }
+        }
+
+        ctx.state.proto_request = Some(
+            crate::routers::grpc::proto_wrapper::ProtoRequest::Generate(proto_request),
+        );
+        Ok(None)
+    }
+
+    fn name(&self) -> &'static str {
+        "GenerateRequestBuilding"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/generate/response_processing.rs

@@ -0,0 +1,145 @@
+//! Generate response processing stage: Handles both streaming and non-streaming responses
+
+use std::{sync::Arc, time::Instant};
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use crate::{
+    core::AttachedBody,
+    routers::{
+        error,
+        grpc::{
+            common::stages::PipelineStage,
+            context::{FinalResponse, RequestContext},
+            regular::{processor, streaming},
+        },
+    },
+};
+
+/// Generate response processing stage
+///
+/// Extracts generate-specific response processing logic from the old unified ResponseProcessingStage.
+pub(crate) struct GenerateResponseProcessingStage {
+    processor: processor::ResponseProcessor,
+    streaming_processor: Arc<streaming::StreamingProcessor>,
+}
+
+impl GenerateResponseProcessingStage {
+    pub fn new(
+        processor: processor::ResponseProcessor,
+        streaming_processor: Arc<streaming::StreamingProcessor>,
+    ) -> Self {
+        Self {
+            processor,
+            streaming_processor,
+        }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for GenerateResponseProcessingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        self.process_generate_response(ctx).await
+    }
+
+    fn name(&self) -> &'static str {
+        "GenerateResponseProcessing"
+    }
+}
+
+impl GenerateResponseProcessingStage {
+    async fn process_generate_response(
+        &self,
+        ctx: &mut RequestContext,
+    ) -> Result<Option<Response>, Response> {
+        let start_time = Instant::now();
+        let is_streaming = ctx.is_streaming();
+
+        // Extract execution result
+        let execution_result = ctx.state.response.execution_result.take().ok_or_else(|| {
+            error!(
+                function = "GenerateResponseProcessingStage::execute",
+                "No execution result"
+            );
+            error::internal_error("no_execution_result", "No execution result")
+        })?;
+
+        // Get dispatch metadata (needed by both streaming and non-streaming)
+        let dispatch = ctx
+            .state
+            .dispatch
+            .as_ref()
+            .ok_or_else(|| {
+                error!(
+                    function = "GenerateResponseProcessingStage::execute",
+                    "Dispatch metadata not set"
+                );
+                error::internal_error("dispatch_metadata_not_set", "Dispatch metadata not set")
+            })?
+            .clone();
+
+        // Get cached tokenizer (resolved once in preparation stage)
+        let tokenizer = ctx.tokenizer_arc().ok_or_else(|| {
+            error!(
+                function = "GenerateResponseProcessingStage::process_generate_response",
+                "Tokenizer not cached in context"
+            );
+            error::internal_error(
+                "tokenizer_not_cached",
+                "Tokenizer not cached in context - preparation stage may have been skipped",
+            )
+        })?;
+
+        if is_streaming {
+            // Streaming: Use StreamingProcessor and return SSE response
+            let response = self.streaming_processor.clone().process_streaming_generate(
+                execution_result,
+                ctx.generate_request_arc(), // Cheap Arc clone (8 bytes)
+                dispatch,
+                tokenizer,
+            );
+
+            // Attach load guards to response body for proper RAII lifecycle
+            let response = match ctx.state.load_guards.take() {
+                Some(guards) => AttachedBody::wrap_response(response, guards),
+                None => response,
+            };
+
+            return Ok(Some(response));
+        }
+
+        // Non-streaming: Delegate to ResponseProcessor
+        let request_logprobs = ctx.generate_request().return_logprob.unwrap_or(false);
+        let generate_request = ctx.generate_request_arc();
+
+        let stop_decoder = ctx.state.response.stop_decoder.as_mut().ok_or_else(|| {
+            error!(
+                function = "GenerateResponseProcessingStage::execute",
+                "Stop decoder not initialized"
+            );
+            error::internal_error(
+                "stop_decoder_not_initialized",
+                "Stop decoder not initialized",
+            )
+        })?;
+
+        let result_array = self
+            .processor
+            .process_non_streaming_generate_response(
+                execution_result,
+                generate_request,
+                dispatch,
+                stop_decoder,
+                request_logprobs,
+                start_time,
+            )
+            .await?;
+
+        // Store the final response
+        ctx.state.response.final_response = Some(FinalResponse::Generate(result_array));
+
+        Ok(None)
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/mod.rs

@@ -0,0 +1,16 @@
+//! Pipeline stages for regular (non-harmony) model processing
+//!
+//! This module defines stages specific to regular tokenizer-based models.
+
+pub(crate) mod chat;
+pub(crate) mod classify;
+pub(crate) mod embedding;
+pub(crate) mod generate;
+pub(crate) mod preparation;
+pub(crate) mod request_building;
+pub(crate) mod response_processing;
+
+// Re-export main stages used by pipeline
+pub(crate) use preparation::PreparationStage;
+pub(crate) use request_building::RequestBuildingStage;
+pub(crate) use response_processing::ResponseProcessingStage;

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/preparation.rs

@@ -0,0 +1,70 @@
+//! Preparation stage that delegates to endpoint-specific implementations
+//!
+//! This stage checks RequestType at runtime and delegates to the appropriate
+//! endpoint-specific stage (ChatPreparationStage or GeneratePreparationStage).
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use super::{
+    chat::ChatPreparationStage, embedding::preparation::EmbeddingPreparationStage,
+    generate::GeneratePreparationStage,
+};
+use crate::routers::{
+    error as grpc_error,
+    grpc::{
+        common::stages::PipelineStage,
+        context::{RequestContext, RequestType},
+    },
+};
+
+/// Preparation stage (delegates to endpoint-specific implementations)
+pub(crate) struct PreparationStage {
+    chat_stage: ChatPreparationStage,
+    generate_stage: GeneratePreparationStage,
+    embedding_stage: EmbeddingPreparationStage,
+}
+
+impl PreparationStage {
+    pub fn new() -> Self {
+        Self {
+            chat_stage: ChatPreparationStage,
+            generate_stage: GeneratePreparationStage,
+            embedding_stage: EmbeddingPreparationStage::new(),
+        }
+    }
+}
+
+impl Default for PreparationStage {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[async_trait]
+impl PipelineStage for PreparationStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        match &ctx.input.request_type {
+            RequestType::Chat(_) => self.chat_stage.execute(ctx).await,
+            RequestType::Generate(_) => self.generate_stage.execute(ctx).await,
+            RequestType::Embedding(_) => self.embedding_stage.execute(ctx).await,
+            // Classify reuses the embedding preparation (tokenization)
+            RequestType::Classify(_) => self.embedding_stage.execute(ctx).await,
+            RequestType::Responses(_) => {
+                error!(
+                    function = "PreparationStage::execute",
+                    "RequestType::Responses reached regular preparation stage"
+                );
+                Err(grpc_error::internal_error(
+                    "responses_in_wrong_pipeline",
+                    "RequestType::Responses reached regular preparation stage",
+                ))
+            }
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "Preparation"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/request_building.rs

@@ -0,0 +1,60 @@
+//! Request building stage that delegates to endpoint-specific implementations
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use super::{
+    chat::ChatRequestBuildingStage, embedding::request_building::EmbeddingRequestBuildingStage,
+    generate::GenerateRequestBuildingStage,
+};
+use crate::routers::{
+    error as grpc_error,
+    grpc::{
+        common::stages::PipelineStage,
+        context::{RequestContext, RequestType},
+    },
+};
+
+/// Request building stage (delegates to endpoint-specific implementations)
+pub(crate) struct RequestBuildingStage {
+    chat_stage: ChatRequestBuildingStage,
+    generate_stage: GenerateRequestBuildingStage,
+    embedding_stage: EmbeddingRequestBuildingStage,
+}
+
+impl RequestBuildingStage {
+    pub fn new(inject_pd_metadata: bool) -> Self {
+        Self {
+            chat_stage: ChatRequestBuildingStage::new(inject_pd_metadata),
+            generate_stage: GenerateRequestBuildingStage::new(inject_pd_metadata),
+            embedding_stage: EmbeddingRequestBuildingStage::new(),
+        }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for RequestBuildingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        match &ctx.input.request_type {
+            RequestType::Chat(_) => self.chat_stage.execute(ctx).await,
+            RequestType::Generate(_) => self.generate_stage.execute(ctx).await,
+            RequestType::Embedding(_) => self.embedding_stage.execute(ctx).await,
+            RequestType::Classify(_) => self.embedding_stage.execute(ctx).await,
+            RequestType::Responses(_request) => {
+                error!(
+                    function = "RequestBuildingStage::execute",
+                    "RequestType::Responses reached regular request building stage"
+                );
+                Err(grpc_error::internal_error(
+                    "responses_in_wrong_pipeline",
+                    "RequestType::Responses reached regular request building stage",
+                ))
+            }
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "RequestBuilding"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/regular/stages/response_processing.rs

@@ -0,0 +1,72 @@
+//! Response processing stage that delegates to endpoint-specific implementations
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use axum::response::Response;
+use tracing::error;
+
+use super::{
+    chat::ChatResponseProcessingStage, classify::ClassifyResponseProcessingStage,
+    embedding::response_processing::EmbeddingResponseProcessingStage,
+    generate::GenerateResponseProcessingStage,
+};
+use crate::routers::{
+    error,
+    grpc::{
+        common::stages::PipelineStage,
+        context::{RequestContext, RequestType},
+        regular::{processor, streaming},
+    },
+};
+
+/// Response processing stage (delegates to endpoint-specific implementations)
+pub(crate) struct ResponseProcessingStage {
+    chat_stage: ChatResponseProcessingStage,
+    generate_stage: GenerateResponseProcessingStage,
+    embedding_stage: EmbeddingResponseProcessingStage,
+    classify_stage: ClassifyResponseProcessingStage,
+}
+
+impl ResponseProcessingStage {
+    pub fn new(
+        processor: processor::ResponseProcessor,
+        streaming_processor: Arc<streaming::StreamingProcessor>,
+    ) -> Self {
+        Self {
+            chat_stage: ChatResponseProcessingStage::new(
+                processor.clone(),
+                streaming_processor.clone(),
+            ),
+            generate_stage: GenerateResponseProcessingStage::new(processor, streaming_processor),
+            embedding_stage: EmbeddingResponseProcessingStage::new(),
+            classify_stage: ClassifyResponseProcessingStage::new(),
+        }
+    }
+}
+
+#[async_trait]
+impl PipelineStage for ResponseProcessingStage {
+    async fn execute(&self, ctx: &mut RequestContext) -> Result<Option<Response>, Response> {
+        match &ctx.input.request_type {
+            RequestType::Chat(_) => self.chat_stage.execute(ctx).await,
+            RequestType::Generate(_) => self.generate_stage.execute(ctx).await,
+            RequestType::Embedding(_) => self.embedding_stage.execute(ctx).await,
+            RequestType::Classify(_) => self.classify_stage.execute(ctx).await,
+            RequestType::Responses(_) => {
+                error!(
+                    function = "ResponseProcessingStage::execute",
+                    "RequestType::Responses reached regular response processing stage"
+                );
+                Err(error::internal_error(
+                    "responses_in_wrong_pipeline",
+                    "RequestType::Responses reached regular response processing stage",
+                ))
+            }
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        "ResponseProcessing"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/grpc/router.rs

@@ -0,0 +1,438 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use axum::{
+    http::HeaderMap,
+    response::{IntoResponse, Response},
+};
+use tracing::debug;
+
+use super::{
+    common::responses::{
+        handlers::{cancel_response_impl, get_response_impl},
+        utils::validate_worker_availability,
+        ResponsesContext,
+    },
+    context::SharedComponents,
+    harmony::{serve_harmony_responses, serve_harmony_responses_stream, HarmonyDetector},
+    pipeline::RequestPipeline,
+    regular::responses,
+};
+use crate::{
+    app_context::AppContext,
+    config::types::RetryConfig,
+    core::{is_retryable_status, RetryExecutor, WorkerRegistry, UNKNOWN_MODEL_ID},
+    observability::metrics::{metrics_labels, Metrics},
+    protocols::{
+        chat::ChatCompletionRequest,
+        classify::ClassifyRequest,
+        embedding::EmbeddingRequest,
+        generate::GenerateRequest,
+        responses::{ResponsesGetParams, ResponsesRequest},
+    },
+    routers::RouterTrait,
+};
+
+/// gRPC router implementation for SGLang
+#[derive(Clone)]
+pub struct GrpcRouter {
+    worker_registry: Arc<WorkerRegistry>,
+    pipeline: RequestPipeline,
+    harmony_pipeline: RequestPipeline,
+    embedding_pipeline: RequestPipeline,
+    classify_pipeline: RequestPipeline,
+    shared_components: Arc<SharedComponents>,
+    responses_context: ResponsesContext,
+    harmony_responses_context: ResponsesContext,
+    retry_config: RetryConfig,
+}
+
+impl GrpcRouter {
+    /// Create a new gRPC router
+    pub async fn new(ctx: &Arc<AppContext>) -> Result<Self, String> {
+        // Get tokenizer registry (no longer requires pre-loaded tokenizer)
+        let tokenizer_registry = ctx.tokenizer_registry.clone();
+
+        let reasoning_parser_factory = ctx
+            .reasoning_parser_factory
+            .as_ref()
+            .ok_or_else(|| "gRPC router requires reasoning parser factory".to_string())?
+            .clone();
+        let tool_parser_factory = ctx
+            .tool_parser_factory
+            .as_ref()
+            .ok_or_else(|| "gRPC router requires tool parser factory".to_string())?
+            .clone();
+
+        let worker_registry = ctx.worker_registry.clone();
+        let _policy_registry = ctx.policy_registry.clone();
+
+        // Create shared components for pipeline
+        let shared_components = Arc::new(SharedComponents {
+            tokenizer_registry: tokenizer_registry.clone(),
+            tool_parser_factory: tool_parser_factory.clone(),
+            reasoning_parser_factory: reasoning_parser_factory.clone(),
+        });
+
+        // Create regular pipeline
+        let pipeline = RequestPipeline::new_regular(
+            worker_registry.clone(),
+            _policy_registry.clone(),
+            tool_parser_factory.clone(),
+            reasoning_parser_factory.clone(),
+            ctx.configured_tool_parser.clone(),
+            ctx.configured_reasoning_parser.clone(),
+        );
+
+        // Create Harmony pipelines
+        let harmony_pipeline = RequestPipeline::new_harmony(
+            worker_registry.clone(),
+            _policy_registry.clone(),
+            tool_parser_factory.clone(),
+            reasoning_parser_factory.clone(),
+            ctx.configured_tool_parser.clone(),
+            ctx.configured_reasoning_parser.clone(),
+        );
+
+        // Create Embedding pipeline
+        let embedding_pipeline =
+            RequestPipeline::new_embeddings(worker_registry.clone(), _policy_registry.clone());
+
+        // Create Classify pipeline
+        let classify_pipeline =
+            RequestPipeline::new_classify(worker_registry.clone(), _policy_registry.clone());
+
+        // Extract shared dependencies for responses contexts
+        let mcp_manager = ctx
+            .mcp_manager
+            .get()
+            .ok_or_else(|| "gRPC router requires MCP manager".to_string())?
+            .clone();
+
+        // Helper closure to create responses context with a given pipeline
+        let create_responses_context = |pipeline: &RequestPipeline| {
+            ResponsesContext::new(
+                Arc::new(pipeline.clone()),
+                shared_components.clone(),
+                ctx.response_storage.clone(),
+                ctx.conversation_storage.clone(),
+                ctx.conversation_item_storage.clone(),
+                mcp_manager.clone(),
+            )
+        };
+
+        // Create responses contexts for both pipelines
+        let responses_context = create_responses_context(&pipeline);
+        let harmony_responses_context = create_responses_context(&harmony_pipeline);
+
+        Ok(GrpcRouter {
+            worker_registry,
+            pipeline,
+            harmony_pipeline,
+            embedding_pipeline,
+            classify_pipeline,
+            shared_components,
+            responses_context,
+            harmony_responses_context,
+            retry_config: ctx.router_config.effective_retry_config(),
+        })
+    }
+
+    /// Main route_chat implementation
+    async fn route_chat_impl(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ChatCompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        // Choose Harmony pipeline if workers indicate Harmony (checks architectures, hf_model_type)
+        let is_harmony =
+            HarmonyDetector::is_harmony_model_in_registry(&self.worker_registry, &body.model);
+
+        debug!(
+            "Processing chat completion request for model: {}, using_harmony={}",
+            model_id.unwrap_or(UNKNOWN_MODEL_ID),
+            is_harmony
+        );
+
+        let pipeline = if is_harmony {
+            &self.harmony_pipeline
+        } else {
+            &self.pipeline
+        };
+
+        // Clone values needed for retry closure
+        let request = Arc::new(body.clone());
+        let headers_cloned = headers.cloned();
+        let model_id_cloned = model_id.map(|s| s.to_string());
+        let components = self.shared_components.clone();
+
+        RetryExecutor::execute_response_with_retry(
+            &self.retry_config,
+            // Operation: execute pipeline (creates fresh context each attempt)
+            |_attempt| {
+                let request = Arc::clone(&request);
+                let headers = headers_cloned.clone();
+                let model_id = model_id_cloned.clone();
+                let components = Arc::clone(&components);
+                async move {
+                    pipeline
+                        .execute_chat(request, headers, model_id, components)
+                        .await
+                }
+            },
+            // Should retry: check if status is retryable
+            |res, _attempt| is_retryable_status(res.status()),
+            // On backoff: record retry metrics
+            |delay, attempt| {
+                Metrics::record_worker_retry(
+                    metrics_labels::WORKER_REGULAR,
+                    metrics_labels::ENDPOINT_CHAT,
+                );
+                Metrics::record_worker_retry_backoff(attempt, delay);
+            },
+            // On exhausted: record exhaustion
+            || {
+                Metrics::record_worker_retries_exhausted(
+                    metrics_labels::WORKER_REGULAR,
+                    metrics_labels::ENDPOINT_CHAT,
+                );
+            },
+        )
+        .await
+    }
+
+    /// Main route_generate implementation
+    async fn route_generate_impl(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &GenerateRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        debug!(
+            "Processing generate request for model: {}",
+            model_id.unwrap_or(UNKNOWN_MODEL_ID)
+        );
+
+        // Clone values needed for retry closure
+        let request = Arc::new(body.clone());
+        let headers_cloned = headers.cloned();
+        let model_id_cloned = model_id.map(|s| s.to_string());
+        let components = self.shared_components.clone();
+        let pipeline = &self.pipeline;
+
+        RetryExecutor::execute_response_with_retry(
+            &self.retry_config,
+            // Operation: execute pipeline (creates fresh context each attempt)
+            |_attempt| {
+                let request = Arc::clone(&request);
+                let headers = headers_cloned.clone();
+                let model_id = model_id_cloned.clone();
+                let components = Arc::clone(&components);
+                async move {
+                    pipeline
+                        .execute_generate(request, headers, model_id, components)
+                        .await
+                }
+            },
+            // Should retry: check if status is retryable
+            |res, _attempt| is_retryable_status(res.status()),
+            // On backoff: record retry metrics
+            |delay, attempt| {
+                Metrics::record_worker_retry(
+                    metrics_labels::WORKER_REGULAR,
+                    metrics_labels::ENDPOINT_GENERATE,
+                );
+                Metrics::record_worker_retry_backoff(attempt, delay);
+            },
+            // On exhausted: record exhaustion
+            || {
+                Metrics::record_worker_retries_exhausted(
+                    metrics_labels::WORKER_REGULAR,
+                    metrics_labels::ENDPOINT_GENERATE,
+                );
+            },
+        )
+        .await
+    }
+
+    /// Main route_responses implementation
+    ///
+    /// Routes to either Harmony or regular responses implementation based on model detection
+    async fn route_responses_impl(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ResponsesRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        // 0. Fast worker validation (fail-fast before expensive operations)
+        let requested_model: Option<&str> = model_id.or(Some(body.model.as_str()));
+
+        if let Some(error_response) = requested_model
+            .and_then(|model| validate_worker_availability(&self.worker_registry, model))
+        {
+            return error_response;
+        }
+
+        // Choose implementation based on Harmony model detection (checks worker metadata)
+        let is_harmony =
+            HarmonyDetector::is_harmony_model_in_registry(&self.worker_registry, &body.model);
+
+        if is_harmony {
+            debug!(
+                "Processing Harmony responses request for model: {}, streaming: {}",
+                model_id.unwrap_or(UNKNOWN_MODEL_ID),
+                body.stream.unwrap_or(false)
+            );
+            let harmony_ctx = ResponsesContext::new(
+                Arc::new(self.harmony_pipeline.clone()),
+                self.shared_components.clone(),
+                self.harmony_responses_context.response_storage.clone(),
+                self.harmony_responses_context.conversation_storage.clone(),
+                self.harmony_responses_context
+                    .conversation_item_storage
+                    .clone(),
+                self.harmony_responses_context.mcp_manager.clone(),
+            );
+
+            if body.stream.unwrap_or(false) {
+                serve_harmony_responses_stream(&harmony_ctx, body.clone()).await
+            } else {
+                match serve_harmony_responses(&harmony_ctx, body.clone()).await {
+                    Ok(response) => axum::Json(response).into_response(),
+                    Err(error_response) => error_response,
+                }
+            }
+        } else {
+            responses::route_responses(
+                &self.responses_context,
+                Arc::new(body.clone()),
+                headers.cloned(),
+                model_id.map(|s| s.to_string()),
+            )
+            .await
+        }
+    }
+
+    /// Main route_embeddings implementation
+    async fn route_embeddings_impl(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &EmbeddingRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        debug!(
+            "Processing embedding request for model: {}",
+            model_id.unwrap_or(UNKNOWN_MODEL_ID)
+        );
+
+        self.embedding_pipeline
+            .execute_embeddings(
+                Arc::new(body.clone()),
+                headers.cloned(),
+                model_id.map(|s| s.to_string()),
+                self.shared_components.clone(),
+            )
+            .await
+    }
+
+    /// Main route_classify implementation
+    async fn route_classify_impl(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ClassifyRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        debug!(
+            "Processing classify request for model: {}",
+            model_id.unwrap_or(UNKNOWN_MODEL_ID)
+        );
+
+        self.classify_pipeline
+            .execute_classify(
+                Arc::new(body.clone()),
+                headers.cloned(),
+                model_id.map(|s| s.to_string()),
+                self.shared_components.clone(),
+            )
+            .await
+    }
+}
+
+impl std::fmt::Debug for GrpcRouter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let stats = self.worker_registry.stats();
+        f.debug_struct("GrpcRouter")
+            .field("workers_count", &stats.total_workers)
+            .finish()
+    }
+}
+
+#[async_trait]
+impl RouterTrait for GrpcRouter {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    async fn route_generate(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &GenerateRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_generate_impl(headers, body, model_id).await
+    }
+
+    async fn route_chat(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ChatCompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_chat_impl(headers, body, model_id).await
+    }
+
+    async fn route_responses(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ResponsesRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_responses_impl(headers, body, model_id).await
+    }
+
+    async fn get_response(
+        &self,
+        _headers: Option<&HeaderMap>,
+        response_id: &str,
+        _params: &ResponsesGetParams,
+    ) -> Response {
+        get_response_impl(&self.responses_context, response_id).await
+    }
+
+    async fn cancel_response(&self, _headers: Option<&HeaderMap>, response_id: &str) -> Response {
+        cancel_response_impl(&self.responses_context, response_id).await
+    }
+
+    async fn route_embeddings(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &EmbeddingRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_embeddings_impl(headers, body, model_id).await
+    }
+
+    async fn route_classify(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ClassifyRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_classify_impl(headers, body, model_id).await
+    }
+
+    fn router_type(&self) -> &'static str {
+        "grpc"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/header_utils.rs

@@ -0,0 +1,299 @@
+use axum::{
+    body::Body,
+    extract::Request,
+    http::{HeaderMap, HeaderValue},
+};
+use http::header::HeaderName;
+
+static HEADER_TARGET_WORKER: HeaderName = HeaderName::from_static("x-smg-target-worker");
+static HEADER_ROUTING_KEY: HeaderName = HeaderName::from_static("x-smg-routing-key");
+
+fn extract_header_value<'a>(headers: Option<&'a HeaderMap>, name: &HeaderName) -> Option<&'a str> {
+    headers
+        .and_then(|h| h.get(name))
+        .and_then(|v| v.to_str().ok())
+        .filter(|s| !s.is_empty())
+}
+
+pub fn extract_target_worker(headers: Option<&HeaderMap>) -> Option<&str> {
+    extract_header_value(headers, &HEADER_TARGET_WORKER)
+}
+
+pub fn extract_routing_key(headers: Option<&HeaderMap>) -> Option<&str> {
+    extract_header_value(headers, &HEADER_ROUTING_KEY)
+}
+
+/// Copy request headers to a Vec of name-value string pairs
+/// Used for forwarding headers to backend workers
+pub fn copy_request_headers(req: &Request<Body>) -> Vec<(String, String)> {
+    req.headers()
+        .iter()
+        .filter_map(|(name, value)| {
+            // Convert header value to string, skipping non-UTF8 headers
+            value
+                .to_str()
+                .ok()
+                .map(|v| (name.to_string(), v.to_string()))
+        })
+        .collect()
+}
+
+/// Convert headers from reqwest Response to axum HeaderMap
+/// Filters out hop-by-hop headers that shouldn't be forwarded
+pub fn preserve_response_headers(reqwest_headers: &HeaderMap) -> HeaderMap {
+    let mut headers = HeaderMap::new();
+
+    for (name, value) in reqwest_headers.iter() {
+        // Skip hop-by-hop headers that shouldn't be forwarded
+        // Use eq_ignore_ascii_case to avoid string allocation
+        if should_forward_header_no_alloc(name.as_str()) {
+            // The original name and value are already valid, so we can just clone them
+            headers.insert(name.clone(), value.clone());
+        }
+    }
+
+    headers
+}
+
+/// Determine if a header should be forwarded without allocating (case-insensitive)
+fn should_forward_header_no_alloc(name: &str) -> bool {
+    // List of headers that should NOT be forwarded (hop-by-hop headers)
+    // Use eq_ignore_ascii_case to avoid to_lowercase() allocation
+    !(name.eq_ignore_ascii_case("connection")
+        || name.eq_ignore_ascii_case("keep-alive")
+        || name.eq_ignore_ascii_case("proxy-authenticate")
+        || name.eq_ignore_ascii_case("proxy-authorization")
+        || name.eq_ignore_ascii_case("te")
+        || name.eq_ignore_ascii_case("trailers")
+        || name.eq_ignore_ascii_case("transfer-encoding")
+        || name.eq_ignore_ascii_case("upgrade")
+        || name.eq_ignore_ascii_case("content-encoding")
+        || name.eq_ignore_ascii_case("host"))
+}
+
+/// Apply headers to a reqwest request builder, filtering out headers that shouldn't be forwarded
+/// or that will be set automatically by reqwest
+pub fn apply_request_headers(
+    headers: &HeaderMap,
+    mut request_builder: reqwest::RequestBuilder,
+    skip_content_headers: bool,
+) -> reqwest::RequestBuilder {
+    // Always forward Authorization header first if present
+    if let Some(auth) = headers
+        .get("authorization")
+        .or_else(|| headers.get("Authorization"))
+    {
+        request_builder = request_builder.header("Authorization", auth.clone());
+    }
+
+    // Forward other headers, filtering out problematic ones
+    // Use eq_ignore_ascii_case to avoid to_lowercase() allocation per header
+    for (key, value) in headers.iter() {
+        let key_str = key.as_str();
+
+        // Skip headers that:
+        // - Are set automatically by reqwest (content-type, content-length for POST/PUT)
+        // - We already handled (authorization)
+        // - Are hop-by-hop headers (connection, transfer-encoding)
+        // - Should not be forwarded (host)
+        let should_skip = key_str.eq_ignore_ascii_case("authorization") // Already handled above
+            || key_str.eq_ignore_ascii_case("host")
+            || key_str.eq_ignore_ascii_case("connection")
+            || key_str.eq_ignore_ascii_case("transfer-encoding")
+            || key_str.eq_ignore_ascii_case("keep-alive")
+            || key_str.eq_ignore_ascii_case("te")
+            || key_str.eq_ignore_ascii_case("trailers")
+            || key_str.eq_ignore_ascii_case("accept-encoding")
+            || key_str.eq_ignore_ascii_case("upgrade")
+            || (skip_content_headers
+                && (key_str.eq_ignore_ascii_case("content-type")
+                    || key_str.eq_ignore_ascii_case("content-length")));
+
+        if !should_skip {
+            request_builder = request_builder.header(key.clone(), value.clone());
+        }
+    }
+
+    request_builder
+}
+
+/// API provider types for provider-specific header handling
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ApiProvider {
+    Anthropic,
+    Xai,
+    OpenAi,
+    Gemini,
+    Generic,
+}
+
+impl ApiProvider {
+    /// Detect provider type from URL
+    pub fn from_url(url: &str) -> Self {
+        if url.contains("anthropic") {
+            ApiProvider::Anthropic
+        } else if url.contains("x.ai") {
+            ApiProvider::Xai
+        } else if url.contains("openai.com") {
+            ApiProvider::OpenAi
+        } else if url.contains("googleapis.com") {
+            ApiProvider::Gemini
+        } else {
+            ApiProvider::Generic
+        }
+    }
+}
+
+/// Apply provider-specific headers to request
+pub fn apply_provider_headers(
+    mut req: reqwest::RequestBuilder,
+    url: &str,
+    auth_header: Option<&HeaderValue>,
+) -> reqwest::RequestBuilder {
+    let provider = ApiProvider::from_url(url);
+
+    match provider {
+        ApiProvider::Anthropic => {
+            // Anthropic requires x-api-key instead of Authorization
+            // Extract Bearer token and use as x-api-key
+            if let Some(auth) = auth_header {
+                if let Ok(auth_str) = auth.to_str() {
+                    let api_key = auth_str.strip_prefix("Bearer ").unwrap_or(auth_str);
+                    req = req
+                        .header("x-api-key", api_key)
+                        .header("anthropic-version", "2023-06-01");
+                }
+            }
+        }
+        ApiProvider::Gemini | ApiProvider::Xai | ApiProvider::OpenAi | ApiProvider::Generic => {
+            // Standard OpenAI-compatible: use Authorization header as-is
+            if let Some(auth) = auth_header {
+                req = req.header("Authorization", auth);
+            }
+        }
+    }
+
+    req
+}
+
+/// Extract auth header with passthrough semantics.
+///
+/// Passthrough mode: User's Authorization header takes priority.
+/// Fallback: Worker's API key is used only if user didn't provide auth.
+///
+/// This enables use cases where:
+/// 1. Users send their own API keys (multi-tenant, BYOK)
+/// 2. Router has a default key for users who don't provide one
+pub fn extract_auth_header(
+    headers: Option<&HeaderMap>,
+    worker_api_key: &Option<String>,
+) -> Option<HeaderValue> {
+    // Passthrough: Try user's auth header first
+    let user_auth = headers.and_then(|h| {
+        h.get("authorization")
+            .or_else(|| h.get("Authorization"))
+            .cloned()
+    });
+
+    // Return user's auth if provided, otherwise use worker's API key
+    user_auth.or_else(|| {
+        worker_api_key
+            .as_ref()
+            .and_then(|k| HeaderValue::from_str(&format!("Bearer {}", k)).ok())
+    })
+}
+
+#[inline]
+pub fn should_forward_request_header(name: &str) -> bool {
+    const REQUEST_ID_PREFIX: &str = "x-request-id-";
+
+    name.eq_ignore_ascii_case("authorization")
+        || name.eq_ignore_ascii_case("x-request-id")
+        || name.eq_ignore_ascii_case("x-correlation-id")
+        || name.eq_ignore_ascii_case("traceparent")
+        || name.eq_ignore_ascii_case("tracestate")
+        || name.eq_ignore_ascii_case("x-smg-routing-key")
+        || name
+            .get(..REQUEST_ID_PREFIX.len())
+            .is_some_and(|prefix| prefix.eq_ignore_ascii_case(REQUEST_ID_PREFIX))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_extract_header_value_returns_value() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-smg-routing-key", "test-key".parse().unwrap());
+        assert_eq!(extract_routing_key(Some(&headers)), Some("test-key"));
+    }
+
+    #[test]
+    fn test_extract_header_value_returns_none_for_missing() {
+        let headers = HeaderMap::new();
+        assert_eq!(extract_routing_key(Some(&headers)), None);
+    }
+
+    #[test]
+    fn test_extract_header_value_returns_none_for_empty() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-smg-routing-key", "".parse().unwrap());
+        assert_eq!(extract_routing_key(Some(&headers)), None);
+    }
+
+    #[test]
+    fn test_extract_header_value_returns_none_for_none_headers() {
+        assert_eq!(extract_routing_key(None), None);
+    }
+
+    #[test]
+    fn test_extract_target_worker() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-smg-target-worker", "2".parse().unwrap());
+        assert_eq!(extract_target_worker(Some(&headers)), Some("2"));
+    }
+
+    #[test]
+    fn test_extract_target_worker_missing() {
+        let headers = HeaderMap::new();
+        assert_eq!(extract_target_worker(Some(&headers)), None);
+    }
+
+    #[test]
+    fn test_should_forward_request_header_whitelist() {
+        assert!(should_forward_request_header("authorization"));
+        assert!(should_forward_request_header("Authorization"));
+        assert!(should_forward_request_header("AUTHORIZATION"));
+        assert!(should_forward_request_header("x-request-id"));
+        assert!(should_forward_request_header("X-Request-Id"));
+        assert!(should_forward_request_header("x-correlation-id"));
+        assert!(should_forward_request_header("X-Correlation-ID"));
+        assert!(should_forward_request_header("traceparent"));
+        assert!(should_forward_request_header("Traceparent"));
+        assert!(should_forward_request_header("tracestate"));
+        assert!(should_forward_request_header("Tracestate"));
+        assert!(should_forward_request_header("x-request-id-user"));
+        assert!(should_forward_request_header("X-Request-ID-Span"));
+        assert!(should_forward_request_header("x-request-id-123"));
+        assert!(should_forward_request_header("x-smg-routing-key"));
+        assert!(should_forward_request_header("X-SMG-Routing-Key"));
+    }
+
+    #[test]
+    fn test_should_forward_request_header_blocked() {
+        assert!(!should_forward_request_header("content-type"));
+        assert!(!should_forward_request_header("Content-Type"));
+        assert!(!should_forward_request_header("content-length"));
+        assert!(!should_forward_request_header("host"));
+        assert!(!should_forward_request_header("Host"));
+        assert!(!should_forward_request_header("connection"));
+        assert!(!should_forward_request_header("transfer-encoding"));
+        assert!(!should_forward_request_header("accept"));
+        assert!(!should_forward_request_header("accept-encoding"));
+        assert!(!should_forward_request_header("user-agent"));
+        assert!(!should_forward_request_header("cookie"));
+        assert!(!should_forward_request_header("x-custom-header"));
+        assert!(!should_forward_request_header("x-api-key"));
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/http/mod.rs

@@ -0,0 +1,5 @@
+//! HTTP router implementations
+
+pub mod pd_router;
+pub mod pd_types;
+pub mod router;

third_party/sglang/sgl-model-gateway/src/routers/http/pd_types.rs

@@ -0,0 +1,80 @@
+//! Types and utilities for the prefill-decode (PD) disaggregated router.
+
+/// Custom error type for PD router operations
+#[derive(Debug, thiserror::Error)]
+pub enum PDRouterError {
+    #[error("Worker already exists: {url}")]
+    WorkerAlreadyExists { url: String },
+
+    #[error("Worker not found: {url}")]
+    WorkerNotFound { url: String },
+
+    #[error("Lock acquisition failed: {operation}")]
+    LockError { operation: String },
+
+    #[error("Health check failed for worker: {url}")]
+    HealthCheckFailed { url: String },
+
+    #[error("Invalid worker configuration: {reason}")]
+    InvalidConfiguration { reason: String },
+
+    #[error("Network error: {message}")]
+    NetworkError { message: String },
+
+    #[error("Timeout waiting for worker: {url}")]
+    Timeout { url: String },
+}
+
+/// Construct a full API URL from a base URL and path.
+pub fn api_path(url: &str, api_path: &str) -> String {
+    if api_path.starts_with('/') {
+        format!("{}{}", url, api_path)
+    } else {
+        format!("{}/{}", url, api_path)
+    }
+}
+
+use serde::Serialize;
+
+/// Optimized bootstrap wrapper for single requests.
+#[derive(Serialize)]
+pub struct RequestWithBootstrap<'a, T: Serialize> {
+    #[serde(flatten)]
+    pub original: &'a T,
+    pub bootstrap_host: String,
+    pub bootstrap_port: Option<u16>,
+    pub bootstrap_room: u64,
+}
+
+/// Optimized bootstrap wrapper for batch requests.
+#[derive(Serialize)]
+pub struct BatchRequestWithBootstrap<'a, T: Serialize> {
+    #[serde(flatten)]
+    pub original: &'a T,
+    pub bootstrap_host: Vec<String>,
+    pub bootstrap_port: Vec<Option<u16>>,
+    pub bootstrap_room: Vec<u64>,
+}
+
+/// Generate a random bootstrap room ID.
+pub fn generate_room_id() -> u64 {
+    // Generate a value in the range [0, 2^63 - 1] to match Python's random.randint(0, 2**63 - 1)
+    rand::random::<u64>() & (i64::MAX as u64)
+}
+
+/// PD-specific routing policies.
+#[derive(Debug, Clone, PartialEq)]
+pub enum PDSelectionPolicy {
+    Random,
+    PowerOfTwo,
+    CacheAware {
+        cache_threshold: f32,
+        balance_abs_threshold: usize,
+        balance_rel_threshold: f32,
+    },
+    Bucket {
+        balance_abs_threshold: usize,
+        balance_rel_threshold: f32,
+        bucket_adjust_interval_secs: usize,
+    },
+}

third_party/sglang/sgl-model-gateway/src/routers/http/router.rs

@@ -0,0 +1,915 @@
+use std::{sync::Arc, time::Instant};
+
+use axum::{
+    body::{to_bytes, Body},
+    extract::Request,
+    http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, Method, StatusCode},
+    response::{IntoResponse, Response},
+    Json,
+};
+use futures_util::{stream, StreamExt};
+use reqwest::Client;
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use tracing::{debug, error};
+
+use crate::{
+    app_context::AppContext,
+    config::types::RetryConfig,
+    core::{
+        is_retryable_status, AttachedBody, ConnectionMode, RetryExecutor, Worker, WorkerLoadGuard,
+        WorkerRegistry, WorkerType, UNKNOWN_MODEL_ID,
+    },
+    observability::{
+        events::{self, Event},
+        metrics::{bool_to_static_str, metrics_labels, Metrics},
+        otel_trace::inject_trace_context_http,
+    },
+    policies::{PolicyRegistry, SelectWorkerInfo},
+    protocols::{
+        chat::ChatCompletionRequest,
+        classify::ClassifyRequest,
+        common::GenerationRequest,
+        completion::CompletionRequest,
+        embedding::EmbeddingRequest,
+        generate::GenerateRequest,
+        rerank::{RerankRequest, RerankResponse, RerankResult},
+        responses::{ResponsesGetParams, ResponsesRequest},
+    },
+    routers::{
+        error::{self, extract_error_code_from_response},
+        grpc::utils::{error_type_from_status, route_to_endpoint},
+        header_utils, RouterTrait,
+    },
+};
+
+/// Regular router that uses injected load balancing policies
+pub struct Router {
+    worker_registry: Arc<WorkerRegistry>,
+    policy_registry: Arc<PolicyRegistry>,
+    client: Client,
+    dp_aware: bool,
+    enable_igw: bool,
+    retry_config: RetryConfig,
+}
+
+impl std::fmt::Debug for Router {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Router")
+            .field("worker_registry", &self.worker_registry)
+            .field("policy_registry", &self.policy_registry)
+            .field("client", &self.client)
+            .field("dp_aware", &self.dp_aware)
+            .field("enable_igw", &self.enable_igw)
+            .field("retry_config", &self.retry_config)
+            .finish()
+    }
+}
+
+impl Router {
+    /// Create a new router with injected policy and client
+    pub async fn new(ctx: &Arc<AppContext>) -> Result<Self, String> {
+        Ok(Router {
+            worker_registry: ctx.worker_registry.clone(),
+            policy_registry: ctx.policy_registry.clone(),
+            client: ctx.client.clone(),
+            dp_aware: ctx.router_config.dp_aware,
+            enable_igw: ctx.router_config.enable_igw,
+            retry_config: ctx.router_config.effective_retry_config(),
+        })
+    }
+
+    fn select_first_worker(&self) -> Result<String, String> {
+        let workers = self.worker_registry.get_all();
+        let healthy_workers: Vec<_> = workers.iter().filter(|w| w.is_healthy()).collect();
+        if healthy_workers.is_empty() {
+            Err("No workers are available".to_string())
+        } else {
+            Ok(healthy_workers[0].url().to_string())
+        }
+    }
+
+    async fn proxy_get_request(&self, req: Request<Body>, endpoint: &str) -> Response {
+        let headers = header_utils::copy_request_headers(&req);
+
+        match self.select_first_worker() {
+            Ok(worker_url) => {
+                let mut request_builder = self.client.get(format!("{}/{}", worker_url, endpoint));
+                for (name, value) in headers {
+                    if header_utils::should_forward_request_header(&name) {
+                        request_builder = request_builder.header(name, value);
+                    }
+                }
+
+                match request_builder.send().await {
+                    Ok(res) => {
+                        let status = StatusCode::from_u16(res.status().as_u16())
+                            .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
+
+                        // Preserve headers from backend
+                        let response_headers =
+                            header_utils::preserve_response_headers(res.headers());
+
+                        match res.bytes().await {
+                            Ok(body) => {
+                                let mut response = Response::new(Body::from(body));
+                                *response.status_mut() = status;
+                                *response.headers_mut() = response_headers;
+                                response
+                            }
+                            Err(e) => error::internal_error(
+                                "read_response_failed",
+                                format!("Failed to read response: {}", e),
+                            ),
+                        }
+                    }
+                    Err(e) => convert_reqwest_error(e),
+                }
+            }
+            Err(e) => error::service_unavailable("no_workers", e),
+        }
+    }
+
+    /// Select worker for a specific model considering circuit breaker state
+    async fn select_worker_for_model(
+        &self,
+        model_id: Option<&str>,
+        text: Option<&str>,
+        headers: Option<&HeaderMap>,
+    ) -> Option<Arc<dyn Worker>> {
+        let effective_model_id = if !self.enable_igw { None } else { model_id };
+
+        // Get workers for the specified model O(1), filtered by connection mode
+        let workers = self.worker_registry.get_workers_filtered(
+            effective_model_id,
+            Some(WorkerType::Regular),
+            Some(ConnectionMode::Http),
+            None,  // any runtime type
+            false, // get all workers, we'll filter by is_available() next
+        );
+
+        let available: Vec<Arc<dyn Worker>> = workers
+            .iter()
+            .filter(|w| w.is_available())
+            .cloned()
+            .collect();
+        if available.is_empty() {
+            return None;
+        }
+
+        // Get the appropriate policy for this model
+        let policy = match model_id {
+            Some(model) => self.policy_registry.get_policy_or_default(model),
+            None => self.policy_registry.get_default_policy(),
+        };
+
+        // Get cached hash ring for consistent hashing (O(log n) lookup)
+        let hash_ring = self
+            .worker_registry
+            .get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
+
+        let idx = policy
+            .select_worker(
+                &available,
+                &SelectWorkerInfo {
+                    request_text: text,
+                    tokens: None, // HTTP doesn't have tokens, use gRPC for PrefixHash
+                    headers,
+                    hash_ring,
+                },
+            )
+            .await?;
+
+        // Record worker selection metric (Layer 3)
+        Metrics::record_worker_selection(
+            metrics_labels::WORKER_REGULAR,
+            metrics_labels::CONNECTION_HTTP,
+            model_id.unwrap_or(UNKNOWN_MODEL_ID),
+            policy.name(),
+        );
+
+        Some(available[idx].clone())
+    }
+
+    pub async fn route_typed_request<T: GenerationRequest + serde::Serialize + Clone>(
+        &self,
+        headers: Option<&HeaderMap>,
+        typed_req: &T,
+        route: &'static str,
+        model_id: Option<&str>,
+    ) -> Response {
+        let start = Instant::now();
+        let is_stream = typed_req.is_stream();
+        let text = typed_req.extract_text_for_routing();
+        let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
+        let endpoint = route_to_endpoint(route);
+
+        // Record request start (Layer 2)
+        Metrics::record_router_request(
+            metrics_labels::ROUTER_HTTP,
+            metrics_labels::BACKEND_REGULAR,
+            metrics_labels::CONNECTION_HTTP,
+            model,
+            endpoint,
+            bool_to_static_str(is_stream),
+        );
+
+        let response = RetryExecutor::execute_response_with_retry(
+            &self.retry_config,
+            // operation per attempt
+            |_: u32| async {
+                let res = self
+                    .route_typed_request_once(headers, typed_req, route, model_id, is_stream, &text)
+                    .await;
+
+                // Need to be outside `route_typed_request_once` because that function has multiple return paths
+                Metrics::record_router_upstream_response(
+                    metrics_labels::ROUTER_HTTP,
+                    res.status().as_u16(),
+                    extract_error_code_from_response(&res),
+                );
+
+                res
+            },
+            // should_retry predicate
+            |res, _attempt| is_retryable_status(res.status()),
+            // on_backoff hook
+            |delay, attempt| {
+                // Layer 3 worker metrics
+                Metrics::record_worker_retry(metrics_labels::WORKER_REGULAR, endpoint);
+                Metrics::record_worker_retry_backoff(attempt, delay);
+            },
+            // on_exhausted hook
+            || {
+                Metrics::record_worker_retries_exhausted(metrics_labels::WORKER_REGULAR, endpoint);
+            },
+        )
+        .await;
+
+        if response.status().is_success() {
+            let duration = start.elapsed();
+            Metrics::record_router_duration(
+                metrics_labels::ROUTER_HTTP,
+                metrics_labels::BACKEND_REGULAR,
+                metrics_labels::CONNECTION_HTTP,
+                model,
+                endpoint,
+                duration,
+            );
+        } else if !is_retryable_status(response.status()) {
+            Metrics::record_router_error(
+                metrics_labels::ROUTER_HTTP,
+                metrics_labels::BACKEND_REGULAR,
+                metrics_labels::CONNECTION_HTTP,
+                model,
+                endpoint,
+                error_type_from_status(response.status()),
+            );
+        }
+
+        response
+    }
+
+    async fn route_typed_request_once<T: GenerationRequest + serde::Serialize + Clone>(
+        &self,
+        headers: Option<&HeaderMap>,
+        typed_req: &T,
+        route: &'static str,
+        model_id: Option<&str>,
+        is_stream: bool,
+        text: &str,
+    ) -> Response {
+        let worker = match self
+            .select_worker_for_model(model_id, Some(text), headers)
+            .await
+        {
+            Some(w) => w,
+            None => {
+                return error::service_unavailable(
+                    "no_available_workers",
+                    "No available workers (all circuits open or unhealthy)",
+                );
+            }
+        };
+
+        let policy = match model_id {
+            Some(model) => self.policy_registry.get_policy_or_default(model),
+            None => self.policy_registry.get_default_policy(),
+        };
+
+        let load_guard = ["cache_aware", "manual"]
+            .contains(&policy.name())
+            .then(|| WorkerLoadGuard::new(worker.clone(), headers));
+
+        // Note: Using borrowed reference avoids heap allocation
+        events::RequestSentEvent { url: worker.url() }.emit();
+        let mut headers_with_trace = headers.cloned().unwrap_or_default();
+        inject_trace_context_http(&mut headers_with_trace);
+        let headers = Some(&headers_with_trace);
+
+        let response = self
+            .send_typed_request(
+                headers,
+                typed_req,
+                route,
+                worker.url(),
+                is_stream,
+                load_guard,
+            )
+            .await;
+
+        events::RequestReceivedEvent {}.emit();
+
+        let status = response.status();
+        worker.record_outcome(status.is_success());
+
+        // Record worker errors for server errors (5xx)
+        if status.is_server_error() {
+            Metrics::record_worker_error(
+                metrics_labels::WORKER_REGULAR,
+                metrics_labels::CONNECTION_HTTP,
+                error_type_from_status(status),
+            );
+        }
+
+        response
+    }
+
+    // Helper: return base worker URL (strips DP suffix when enabled)
+    fn worker_base_url(&self, worker_url: &str) -> String {
+        if self.dp_aware {
+            if let Ok((prefix, _)) = Self::extract_dp_rank(worker_url) {
+                return prefix.to_string();
+            }
+        }
+        worker_url.to_string()
+    }
+
+    // Generic simple routing for GET/POST without JSON body
+    async fn route_simple_request(
+        &self,
+        headers: Option<&HeaderMap>,
+        endpoint: &str,
+        method: Method,
+    ) -> Response {
+        // TODO: currently the sglang worker is using in-memory state management, so this implementation has to fan out to all workers.
+        // Eventually, we need to have router to manage the chat history with a proper database, will update this implementation accordingly.
+        let workers = self.worker_registry.get_all();
+        if workers.is_empty() {
+            return error::service_unavailable("no_workers", "No available workers");
+        }
+
+        let filtered_headers: Vec<_> = headers
+            .map(|hdrs| {
+                hdrs.iter()
+                    .filter(|(name, _)| header_utils::should_forward_request_header(name.as_str()))
+                    .collect()
+            })
+            .unwrap_or_default();
+
+        let futures: Vec<_> = workers
+            .into_iter()
+            .map(|worker| {
+                let worker_url = worker.url();
+                let base = self.worker_base_url(worker_url);
+                let url = format!("{}/{}", base, endpoint);
+                let client = self.client.clone();
+                let method = method.clone();
+
+                let headers = filtered_headers.clone();
+
+                let api_key = worker.api_key().clone();
+
+                async move {
+                    let mut request_builder = match method {
+                        Method::GET => client.get(url),
+                        Method::POST => client.post(url),
+                        _ => {
+                            return Err(error::method_not_allowed(
+                                "unsupported_method",
+                                "Unsupported method for simple routing",
+                            ))
+                        }
+                    };
+
+                    if let Some(key) = api_key {
+                        let mut auth_header = String::with_capacity(7 + key.len());
+                        auth_header.push_str("Bearer ");
+                        auth_header.push_str(&key);
+                        request_builder = request_builder.header("Authorization", auth_header);
+                    }
+
+                    for (name, value) in headers {
+                        request_builder = request_builder.header(name.clone(), value.clone());
+                    }
+
+                    request_builder.send().await.map_err(convert_reqwest_error)
+                }
+            })
+            .collect();
+
+        // Now execute the collected futures concurrently
+        let mut stream = stream::iter(futures).buffer_unordered(32);
+        let mut last_response: Option<Response> = None;
+
+        while let Some(result) = stream.next().await {
+            match result {
+                Ok(res) => {
+                    let status = StatusCode::from_u16(res.status().as_u16())
+                        .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
+
+                    let response_headers = header_utils::preserve_response_headers(res.headers());
+
+                    match res.bytes().await {
+                        Ok(body) => {
+                            let mut response = Response::new(Body::from(body));
+                            *response.status_mut() = status;
+                            *response.headers_mut() = response_headers;
+
+                            if status.is_success() {
+                                return response;
+                            }
+                            last_response = Some(response);
+                        }
+                        Err(e) => {
+                            last_response = Some(error::internal_error(
+                                "read_response_failed",
+                                format!("Failed to read response: {}", e),
+                            ));
+                        }
+                    }
+                }
+                Err(e) => {
+                    last_response = Some(e);
+                }
+            }
+        }
+
+        last_response
+            .unwrap_or_else(|| error::bad_gateway("no_worker_response", "No worker response"))
+    }
+
+    // Route a GET request with provided headers to a specific endpoint
+    async fn route_get_request(&self, headers: Option<&HeaderMap>, endpoint: &str) -> Response {
+        self.route_simple_request(headers, endpoint, Method::GET)
+            .await
+    }
+
+    // Route a POST request with empty body to a specific endpoint
+    async fn route_post_empty_request(
+        &self,
+        headers: Option<&HeaderMap>,
+        endpoint: &str,
+    ) -> Response {
+        self.route_simple_request(headers, endpoint, Method::POST)
+            .await
+    }
+
+    // TODO (rui): Better accommodate to the Worker abstraction
+    fn extract_dp_rank(worker_url: &str) -> Result<(&str, usize), String> {
+        let parts: Vec<&str> = worker_url.split('@').collect();
+        if parts.len() != 2 {
+            return Err(format!("invalid worker_url format: {}", worker_url));
+        }
+
+        // Parse the second part (dp_rank) into an integer
+        match parts[1].parse::<usize>() {
+            Ok(dp_rank) => Ok((parts[0], dp_rank)),
+            Err(_) => Err(format!(
+                "failed to parse dp_rank from worker_url: {}",
+                worker_url
+            )),
+        }
+    }
+
+    // Send typed request directly without conversion
+    async fn send_typed_request<T: serde::Serialize>(
+        &self,
+        headers: Option<&HeaderMap>,
+        typed_req: &T,
+        route: &'static str,
+        worker_url: &str,
+        is_stream: bool,
+        load_guard: Option<WorkerLoadGuard>,
+    ) -> Response {
+        // Get the worker once and reuse for API key and load tracking
+        let worker = self.worker_registry.get_by_url(worker_url);
+        let api_key = worker.as_ref().and_then(|w| w.api_key().clone());
+
+        // Static key string to avoid per-request allocations
+        const DP_RANK_KEY: &str = "data_parallel_rank";
+
+        let mut request_builder = if self.dp_aware {
+            let (worker_url_prefix, dp_rank) = match Self::extract_dp_rank(worker_url) {
+                Ok(tup) => tup,
+                Err(e) => {
+                    error!("Failed to extract dp_rank: {}", e);
+                    return error::internal_error(
+                        "dp_rank_extraction_failed",
+                        format!("Failed to extract dp_rank: {}", e),
+                    );
+                }
+            };
+
+            let mut json_val = match serde_json::to_value(typed_req) {
+                Ok(j) => j,
+                Err(e) => {
+                    return error::bad_request(
+                        "serialization_failed",
+                        format!("Convert into serde_json::Value failed: {}", e),
+                    );
+                }
+            };
+
+            if let Some(map) = json_val.as_object_mut() {
+                // Use static key string to avoid allocation
+                map.insert(DP_RANK_KEY.to_string(), serde_json::json!(dp_rank));
+                // Only serialize if debug logging is enabled to avoid CPU overhead
+                if tracing::enabled!(tracing::Level::DEBUG) {
+                    debug!(
+                        "Modified request body: {}",
+                        serde_json::to_string(&json_val).unwrap_or_else(|_| String::from("ERR"))
+                    );
+                }
+            } else {
+                return error::bad_request(
+                    "dp_rank_insertion_failed",
+                    "Failed to insert the data_parallel_rank field into the request body",
+                );
+            }
+
+            self.client
+                .post(format!("{}{}", worker_url_prefix, route))
+                .json(&json_val)
+        } else {
+            self.client
+                .post(format!("{}{}", worker_url, route))
+                .json(typed_req) // Use json() directly with typed request
+        };
+
+        if let Some(key) = api_key {
+            // Pre-allocate string with capacity to avoid reallocation
+            let mut auth_header = String::with_capacity(7 + key.len());
+            auth_header.push_str("Bearer ");
+            auth_header.push_str(&key);
+            request_builder = request_builder.header("Authorization", auth_header);
+        }
+
+        if let Some(headers) = headers {
+            for (name, value) in headers {
+                if header_utils::should_forward_request_header(name.as_str()) {
+                    request_builder = request_builder.header(name, value);
+                }
+            }
+        }
+
+        let res = match request_builder.send().await {
+            Ok(res) => res,
+            Err(e) => {
+                error!(
+                    "Failed to send typed request worker_url={} route={} error={}",
+                    worker_url, route, e
+                );
+
+                return convert_reqwest_error(e);
+            }
+        };
+
+        let status = StatusCode::from_u16(res.status().as_u16())
+            .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
+
+        if !is_stream {
+            // For non-streaming requests, preserve headers
+            let response_headers = header_utils::preserve_response_headers(res.headers());
+
+            let response = match res.bytes().await {
+                Ok(body) => {
+                    let mut response = Response::new(Body::from(body));
+                    *response.status_mut() = status;
+                    *response.headers_mut() = response_headers;
+                    response
+                }
+                Err(e) => {
+                    let error_msg = format!("Failed to get response body: {}", e);
+                    error::internal_error("read_response_body_failed", error_msg)
+                }
+            };
+
+            // load_guard dropped here automatically after response body is read
+            response
+        } else {
+            // Preserve headers for streaming response
+            let mut response_headers = header_utils::preserve_response_headers(res.headers());
+            // Ensure we set the correct content-type for SSE
+            response_headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream"));
+
+            let stream = res.bytes_stream();
+            let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
+
+            // Spawn task to forward stream
+            tokio::spawn(async move {
+                let mut stream = stream;
+                while let Some(chunk) = stream.next().await {
+                    match chunk {
+                        Ok(bytes) => {
+                            if tx.send(Ok(bytes)).is_err() {
+                                break;
+                            }
+                        }
+                        Err(e) => {
+                            let _ = tx.send(Err(format!("Stream error: {}", e)));
+                            break;
+                        }
+                    }
+                }
+            });
+
+            let stream = UnboundedReceiverStream::new(rx);
+            let body = Body::from_stream(stream);
+
+            let mut response = Response::new(body);
+            *response.status_mut() = status;
+            *response.headers_mut() = response_headers;
+
+            // Attach load guard to response body for proper RAII lifecycle
+            // Guard is dropped when response body is consumed or client disconnects
+            if let Some(guard) = load_guard {
+                response = AttachedBody::wrap_response(response, guard);
+            }
+            response
+        }
+    }
+
+    async fn build_rerank_response(
+        req: &RerankRequest,
+        response: Response,
+    ) -> anyhow::Result<Response> {
+        let (_, response_body) = response.into_parts();
+        let body_bytes = to_bytes(response_body, usize::MAX).await?;
+        let rerank_results = serde_json::from_slice::<Vec<RerankResult>>(&body_bytes)?;
+        let mut rerank_response =
+            RerankResponse::new(rerank_results, req.model.clone(), req.rid.clone());
+        // Sorting is handled by Python worker (serving_rerank.py)
+        if let Some(top_k) = req.top_k {
+            rerank_response.apply_top_k(top_k);
+        }
+        if !req.return_documents {
+            rerank_response.drop_documents();
+        }
+        Ok(Json(rerank_response).into_response())
+    }
+}
+
+fn convert_reqwest_error(e: reqwest::Error) -> Response {
+    let url = e
+        .url()
+        .map(|u| u.to_string())
+        .unwrap_or_else(|| "unknown".to_string());
+    let message = format!("{}. URL: {}", e, url);
+
+    // TODO improve error status code
+    let (status, code) = if let Some(upstream_status) = e.status() {
+        (upstream_status, "call_upstream_status_error")
+    } else if e.is_builder() {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "call_upstream_builder_error",
+        )
+    } else if e.is_request() {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "call_upstream_request_error",
+        )
+    } else if e.is_redirect() {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "call_upstream_redirect_error",
+        )
+    } else if e.is_body() {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "call_upstream_body_error",
+        )
+    } else if e.is_decode() {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "call_upstream_decode_error",
+        )
+    } else if e.is_timeout() {
+        (StatusCode::GATEWAY_TIMEOUT, "call_upstream_timeout")
+    } else if e.is_connect() {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "call_upstream_connection_failed",
+        )
+    } else {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "call_upstream_request_failed",
+        )
+    };
+
+    error::create_error(status, code, message)
+}
+
+use async_trait::async_trait;
+
+#[async_trait]
+impl RouterTrait for Router {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    async fn health_generate(&self, req: Request<Body>) -> Response {
+        self.proxy_get_request(req, "health_generate").await
+    }
+
+    async fn get_server_info(&self, req: Request<Body>) -> Response {
+        self.proxy_get_request(req, "server_info").await
+    }
+
+    async fn get_models(&self, req: Request<Body>) -> Response {
+        self.proxy_get_request(req, "v1/models").await
+    }
+
+    async fn get_model_info(&self, req: Request<Body>) -> Response {
+        self.proxy_get_request(req, "get_model_info").await
+    }
+
+    async fn route_generate(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &GenerateRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_typed_request(headers, body, "/generate", model_id)
+            .await
+    }
+
+    async fn route_chat(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ChatCompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_typed_request(headers, body, "/v1/chat/completions", model_id)
+            .await
+    }
+
+    async fn route_completion(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &CompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_typed_request(headers, body, "/v1/completions", model_id)
+            .await
+    }
+
+    async fn route_responses(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ResponsesRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_typed_request(headers, body, "/v1/responses", model_id)
+            .await
+    }
+
+    async fn get_response(
+        &self,
+        headers: Option<&HeaderMap>,
+        response_id: &str,
+        _params: &ResponsesGetParams,
+    ) -> Response {
+        let endpoint = format!("v1/responses/{}", response_id);
+        self.route_get_request(headers, &endpoint).await
+    }
+
+    async fn cancel_response(&self, headers: Option<&HeaderMap>, response_id: &str) -> Response {
+        let endpoint = format!("v1/responses/{}/cancel", response_id);
+        self.route_post_empty_request(headers, &endpoint).await
+    }
+
+    async fn route_embeddings(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &EmbeddingRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_typed_request(headers, body, "/v1/embeddings", model_id)
+            .await
+    }
+
+    async fn route_classify(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ClassifyRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        self.route_typed_request(headers, body, "/v1/classify", model_id)
+            .await
+    }
+
+    async fn route_rerank(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &RerankRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        let response = self
+            .route_typed_request(headers, body, "/v1/rerank", model_id)
+            .await;
+        if response.status().is_success() {
+            match Self::build_rerank_response(body, response).await {
+                Ok(rerank_response) => rerank_response,
+                Err(e) => {
+                    error!("Failed to build rerank response: {}", e);
+                    return error::internal_error(
+                        "rerank_response_build_failed",
+                        "Failed to build rerank response",
+                    );
+                }
+            }
+        } else {
+            response
+        }
+    }
+
+    fn router_type(&self) -> &'static str {
+        "regular"
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::core::BasicWorkerBuilder;
+
+    fn create_test_regular_router() -> Router {
+        // Create registries
+        let worker_registry = Arc::new(WorkerRegistry::new());
+        let policy_registry = Arc::new(PolicyRegistry::new(
+            crate::config::types::PolicyConfig::RoundRobin,
+        ));
+
+        // Register test workers
+        let worker1 = BasicWorkerBuilder::new("http://worker1:8080")
+            .worker_type(WorkerType::Regular)
+            .build();
+        let worker2 = BasicWorkerBuilder::new("http://worker2:8080")
+            .worker_type(WorkerType::Regular)
+            .build();
+        worker_registry.register(Arc::new(worker1));
+        worker_registry.register(Arc::new(worker2));
+
+        Router {
+            worker_registry,
+            policy_registry,
+            dp_aware: false,
+            client: Client::new(),
+            retry_config: RetryConfig::default(),
+            enable_igw: false,
+        }
+    }
+
+    fn create_test_unhealthy_router() -> Router {
+        let router = create_test_regular_router();
+        let workers = router.worker_registry.get_all();
+        workers[0].set_healthy(false);
+        router
+    }
+
+    #[test]
+    fn test_router_get_worker_urls_regular() {
+        let router = create_test_regular_router();
+        let workers = router.worker_registry.get_all();
+        let urls: Vec<String> = workers.iter().map(|w| w.url().to_string()).collect();
+
+        assert_eq!(urls.len(), 2);
+        assert!(urls.contains(&"http://worker1:8080".to_string()));
+        assert!(urls.contains(&"http://worker2:8080".to_string()));
+    }
+
+    #[test]
+    fn test_select_first_worker_regular() {
+        let router = create_test_regular_router();
+        let result = router.select_first_worker();
+
+        assert!(result.is_ok());
+        let url = result.unwrap();
+        // DashMap doesn't guarantee order, so just check we get one of the workers
+        assert!(url == "http://worker1:8080" || url == "http://worker2:8080");
+    }
+
+    #[test]
+    fn test_select_first_worker_with_unhealthy_worker() {
+        let router = create_test_unhealthy_router();
+        let result = router.select_first_worker();
+
+        assert!(result.is_ok());
+        let url = result.unwrap();
+
+        let worker = router.worker_registry.get_by_url(&url).unwrap();
+        assert!(worker.is_healthy());
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/mcp_utils.rs

@@ -0,0 +1,161 @@
+//! Shared MCP utilities for routers.
+//!
+//! This module provides shared MCP-related functionality that can be
+//! used across different router implementations (OpenAI, gRPC regular, gRPC harmony).
+
+use std::sync::Arc;
+
+use smg_mcp::{McpManager, McpServerConfig, McpTransport};
+use tracing::warn;
+
+use crate::protocols::responses::{ResponseTool, ResponseToolType};
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/// Default maximum tool loop iterations (safety limit).
+///
+/// Used as fallback when user doesn't specify `max_tool_calls`.
+/// All routers use this same value.
+pub const DEFAULT_MAX_ITERATIONS: usize = 10;
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+/// Configuration for MCP tool calling loops.
+///
+/// Provides a common structure for loop configuration across routers.
+#[derive(Debug, Clone)]
+pub struct McpLoopConfig {
+    /// Maximum iterations as safety limit (default: DEFAULT_MAX_ITERATIONS).
+    /// Prevents infinite loops when max_tool_calls is not set by user.
+    pub max_iterations: usize,
+    /// Server keys for filtering MCP tools.
+    /// Contains keys for dynamic servers that were connected for this request.
+    pub server_keys: Vec<String>,
+}
+
+impl Default for McpLoopConfig {
+    fn default() -> Self {
+        Self {
+            max_iterations: DEFAULT_MAX_ITERATIONS,
+            server_keys: Vec::new(),
+        }
+    }
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/// Extract MCP server label from request tools.
+///
+/// Searches for the first MCP tool in the tools array and returns its server_label.
+/// Falls back to a default value if no MCP tool with server_label is found.
+pub fn extract_server_label(tools: Option<&[ResponseTool]>, default_label: &str) -> String {
+    tools
+        .and_then(|tools| {
+            tools.iter().find_map(|tool| {
+                if matches!(tool.r#type, ResponseToolType::Mcp) {
+                    tool.server_label.clone()
+                } else {
+                    None
+                }
+            })
+        })
+        .unwrap_or_else(|| default_label.to_string())
+}
+
+// ============================================================================
+// MCP Connection
+// ============================================================================
+
+/// Ensure MCP clients are connected for all request-level MCP tools.
+///
+/// This function extracts MCP server configurations from ALL request tools (server_url, authorization)
+/// and ensures client connections are established via the connection pool.
+///
+/// Returns `Some((manager, server_keys))` if MCP tools were found and clients created,
+/// `None` if no MCP tools with server_url were found.
+pub async fn ensure_request_mcp_client(
+    mcp_manager: &Arc<McpManager>,
+    tools: &[ResponseTool],
+) -> Option<(Arc<McpManager>, Vec<String>)> {
+    let mut server_keys = Vec::new();
+    let mut has_mcp_tools = false;
+
+    // Process all MCP tools
+    for tool in tools {
+        if matches!(tool.r#type, ResponseToolType::Mcp) && tool.server_url.is_some() {
+            has_mcp_tools = true;
+            let Some(server_url) = tool.server_url.as_ref().map(|s| s.trim().to_string()) else {
+                continue;
+            };
+
+            // Validate URL scheme
+            if !(server_url.starts_with("http://") || server_url.starts_with("https://")) {
+                warn!(
+                    "Ignoring MCP server_url with unsupported scheme: {}",
+                    server_url
+                );
+                continue;
+            }
+
+            // Extract server label and auth token
+            let name = tool
+                .server_label
+                .clone()
+                .unwrap_or_else(|| "request-mcp".to_string());
+            let token = tool.authorization.clone();
+
+            // Determine transport type based on URL pattern
+            let transport = if server_url.contains("/sse") {
+                McpTransport::Sse {
+                    url: server_url.clone(),
+                    token,
+                }
+            } else {
+                McpTransport::Streamable {
+                    url: server_url.clone(),
+                    token,
+                }
+            };
+
+            // Create server config
+            let server_config = McpServerConfig {
+                name,
+                transport,
+                proxy: None,
+                required: false,
+            };
+
+            // Get the server key for tracking
+            let server_key = McpManager::server_key(&server_config);
+
+            // Use get_or_create_client to establish connection
+            match mcp_manager.get_or_create_client(server_config).await {
+                Ok(_client) => {
+                    // Track this server for filtering
+                    if !server_keys.contains(&server_key) {
+                        server_keys.push(server_key);
+                    }
+                }
+                Err(err) => {
+                    warn!(
+                        "Failed to get/create MCP connection for {}: {}",
+                        server_key, err
+                    );
+                    // Continue processing other tools
+                }
+            }
+        }
+    }
+
+    if has_mcp_tools && !server_keys.is_empty() {
+        Some((mcp_manager.clone(), server_keys))
+    } else {
+        None
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/mesh/handlers.rs

@@ -0,0 +1,441 @@
+//! Mesh management HTTP handlers
+//!
+//! Provides REST API for mesh cluster management:
+//! - Configuration CRUD operations
+//! - Health checks
+//! - Cluster status
+
+use std::sync::Arc;
+
+use axum::{
+    extract::{Path, State},
+    http::StatusCode,
+    response::{IntoResponse, Response},
+    Json,
+};
+use serde::{Deserialize, Serialize};
+use serde_json::json;
+use smg_mesh::{RateLimitConfig, GLOBAL_RATE_LIMIT_COUNTER_KEY, GLOBAL_RATE_LIMIT_KEY};
+use tracing::{info, warn};
+
+use crate::server::AppState;
+
+/// Mesh cluster status response
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ClusterStatusResponse {
+    pub node_name: String,
+    pub node_count: usize,
+    pub nodes: Vec<NodeInfo>,
+    pub stores: StoreStatus,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct NodeInfo {
+    pub name: String,
+    pub address: String,
+    pub status: String,
+    pub version: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct StoreStatus {
+    pub membership_count: usize,
+    pub worker_count: usize,
+    pub policy_count: usize,
+    pub app_count: usize,
+}
+
+/// Health check response
+#[derive(Debug, Serialize, Deserialize)]
+pub struct MeshHealthResponse {
+    pub status: String,
+    pub node_name: String,
+    pub cluster_size: usize,
+    pub stores_healthy: bool,
+}
+
+/// Get mesh cluster status
+pub async fn get_cluster_status(State(app_state): State<Arc<AppState>>) -> Response {
+    let handler = match &app_state.mesh_handler {
+        Some(h) => h,
+        None => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"error": "mesh not enabled"})),
+            )
+                .into_response();
+        }
+    };
+    let state = handler.state.read();
+    let nodes: Vec<NodeInfo> = state
+        .values()
+        .map(|node| NodeInfo {
+            name: node.name.clone(),
+            address: node.address.clone(),
+            status: format!("{:?}", node.status),
+            version: node.version,
+        })
+        .collect();
+
+    // Get store counts (if stores are available)
+    let stores = StoreStatus {
+        membership_count: state.len(),
+        worker_count: 0, // TODO: Get from stores if available
+        policy_count: 0,
+        app_count: 0,
+    };
+
+    let response = ClusterStatusResponse {
+        node_name: handler.self_name.clone(),
+        node_count: nodes.len(),
+        nodes,
+        stores,
+    };
+
+    (StatusCode::OK, Json(response)).into_response()
+}
+
+/// Get mesh health status
+pub async fn get_mesh_health(State(app_state): State<Arc<AppState>>) -> Response {
+    let handler = match &app_state.mesh_handler {
+        Some(h) => h,
+        None => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"error": "mesh not enabled"})),
+            )
+                .into_response();
+        }
+    };
+    let state = handler.state.read();
+    let cluster_size = state.len();
+
+    let response = MeshHealthResponse {
+        status: "healthy".to_string(),
+        node_name: handler.self_name.clone(),
+        cluster_size,
+        stores_healthy: true, // TODO: Check actual store health
+    };
+
+    (StatusCode::OK, Json(response)).into_response()
+}
+
+/// Get worker states from mesh store
+pub async fn get_worker_states(State(app_state): State<Arc<AppState>>) -> Response {
+    match &app_state.mesh_sync_manager {
+        Some(manager) => {
+            let workers = manager.get_all_worker_states();
+            (StatusCode::OK, Json(workers)).into_response()
+        }
+        None => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({"error": "mesh sync manager not available"})),
+        )
+            .into_response(),
+    }
+}
+
+/// Get policy states from mesh store
+pub async fn get_policy_states(State(app_state): State<Arc<AppState>>) -> Response {
+    match &app_state.mesh_sync_manager {
+        Some(manager) => {
+            let policies = manager.get_all_policy_states();
+            (StatusCode::OK, Json(policies)).into_response()
+        }
+        None => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({"error": "mesh sync manager not available"})),
+        )
+            .into_response(),
+    }
+}
+
+/// Get a specific worker state
+pub async fn get_worker_state(
+    Path(worker_id): Path<String>,
+    State(app_state): State<Arc<AppState>>,
+) -> Response {
+    match &app_state.mesh_sync_manager {
+        Some(manager) => match manager.get_worker_state(&worker_id) {
+            Some(state) => (StatusCode::OK, Json(state)).into_response(),
+            None => (
+                StatusCode::NOT_FOUND,
+                Json(json!({"error": "Worker not found"})),
+            )
+                .into_response(),
+        },
+        None => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({"error": "mesh sync manager not available"})),
+        )
+            .into_response(),
+    }
+}
+
+/// Get a specific policy state
+pub async fn get_policy_state(
+    Path(model_id): Path<String>,
+    State(app_state): State<Arc<AppState>>,
+) -> Response {
+    match &app_state.mesh_sync_manager {
+        Some(manager) => match manager.get_policy_state(&model_id) {
+            Some(state) => (StatusCode::OK, Json(state)).into_response(),
+            None => (
+                StatusCode::NOT_FOUND,
+                Json(json!({"error": "Policy not found"})),
+            )
+                .into_response(),
+        },
+        None => (
+            StatusCode::SERVICE_UNAVAILABLE,
+            Json(json!({"error": "mesh sync manager not available"})),
+        )
+            .into_response(),
+    }
+}
+
+/// Update app configuration
+#[derive(Debug, Deserialize)]
+pub struct UpdateAppConfigRequest {
+    pub key: String,
+    pub value: String, // Hex encoded string
+}
+
+pub async fn update_app_config(
+    State(app_state): State<Arc<AppState>>,
+    Json(request): Json<UpdateAppConfigRequest>,
+) -> Response {
+    let handler = match &app_state.mesh_handler {
+        Some(h) => h,
+        None => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"error": "mesh not enabled"})),
+            )
+                .into_response();
+        }
+    };
+
+    // Decode hex string to bytes
+    // Simple hex decoding without external dependency
+    let value = if request.value.len() % 2 == 0 {
+        match (0..request.value.len())
+            .step_by(2)
+            .map(|i| u8::from_str_radix(&request.value[i..i + 2], 16))
+            .collect::<Result<Vec<u8>, _>>()
+        {
+            Ok(v) => v,
+            Err(_) => {
+                return (
+                    StatusCode::BAD_REQUEST,
+                    Json(json!({"error": "Invalid hex encoding"})),
+                )
+                    .into_response();
+            }
+        }
+    } else {
+        return (
+            StatusCode::BAD_REQUEST,
+            Json(json!({"error": "Hex string must have even length"})),
+        )
+            .into_response();
+    };
+    handler.write_data(request.key.clone(), value);
+    info!("Updated app config: {}", request.key);
+    (
+        StatusCode::OK,
+        Json(json!({"status": "updated", "key": request.key})),
+    )
+        .into_response()
+}
+
+/// Get app configuration
+pub async fn get_app_config(
+    Path(key): Path<String>,
+    State(app_state): State<Arc<AppState>>,
+) -> Response {
+    let handler = match &app_state.mesh_handler {
+        Some(h) => h,
+        None => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"error": "mesh not enabled"})),
+            )
+                .into_response();
+        }
+    };
+    match handler.read_data(key.clone()) {
+        Some(value) => {
+            // Return value as hex encoded string for JSON compatibility
+            let hex_value: String = value.iter().map(|b| format!("{:02x}", b)).collect();
+            (
+                StatusCode::OK,
+                Json(json!({"key": key, "value": hex_value, "format": "hex"})),
+            )
+                .into_response()
+        }
+        None => (
+            StatusCode::NOT_FOUND,
+            Json(json!({"error": "Config not found"})),
+        )
+            .into_response(),
+    }
+}
+
+/// Set global rate limit configuration
+#[derive(Debug, Deserialize)]
+pub struct SetRateLimitRequest {
+    pub limit_per_second: u64,
+}
+
+pub async fn set_global_rate_limit(
+    State(app_state): State<Arc<AppState>>,
+    Json(request): Json<SetRateLimitRequest>,
+) -> Response {
+    // Store configuration in AppStore
+    let config = RateLimitConfig {
+        limit_per_second: request.limit_per_second,
+    };
+
+    if let Ok(config_bytes) = serde_json::to_vec(&config) {
+        let handler = match &app_state.mesh_handler {
+            Some(h) => h,
+            None => {
+                return (
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    Json(json!({"error": "mesh not enabled"})),
+                )
+                    .into_response();
+            }
+        };
+
+        handler.write_data(GLOBAL_RATE_LIMIT_KEY.to_string(), config_bytes);
+        info!("Set global rate limit: {} req/s", request.limit_per_second);
+
+        (
+            StatusCode::OK,
+            Json(json!({
+                "status": "updated",
+                "limit_per_second": request.limit_per_second
+            })),
+        )
+            .into_response()
+    } else {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(json!({"error": "Failed to serialize rate limit config"})),
+        )
+            .into_response()
+    }
+}
+
+/// Get global rate limit configuration
+pub async fn get_global_rate_limit(State(app_state): State<Arc<AppState>>) -> Response {
+    let handler = match &app_state.mesh_handler {
+        Some(h) => h,
+        None => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"error": "mesh not enabled"})),
+            )
+                .into_response();
+        }
+    };
+
+    match handler.read_data(GLOBAL_RATE_LIMIT_KEY.to_string()) {
+        Some(value) => match serde_json::from_slice::<RateLimitConfig>(&value) {
+            Ok(config) => (
+                StatusCode::OK,
+                Json(json!({
+                    "limit_per_second": config.limit_per_second
+                })),
+            )
+                .into_response(),
+            Err(_) => (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(json!({"error": "Failed to deserialize rate limit config"})),
+            )
+                .into_response(),
+        },
+        None => (
+            StatusCode::NOT_FOUND,
+            Json(json!({"error": "Global rate limit not configured"})),
+        )
+            .into_response(),
+    }
+}
+
+/// Get global rate limit statistics
+pub async fn get_global_rate_limit_stats(State(app_state): State<Arc<AppState>>) -> Response {
+    let sync_manager = match &app_state.mesh_sync_manager {
+        Some(m) => m,
+        None => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"error": "mesh sync manager not available"})),
+            )
+                .into_response();
+        }
+    };
+
+    // Get configuration
+    let handler = match &app_state.mesh_handler {
+        Some(h) => h,
+        None => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"error": "mesh not enabled"})),
+            )
+                .into_response();
+        }
+    };
+
+    let config = handler
+        .read_data(GLOBAL_RATE_LIMIT_KEY.to_string())
+        .and_then(|v| serde_json::from_slice::<RateLimitConfig>(&v).ok())
+        .unwrap_or_default();
+
+    // Get current counter value
+    let current_count = sync_manager
+        .get_rate_limit_value(GLOBAL_RATE_LIMIT_COUNTER_KEY)
+        .unwrap_or(0);
+
+    (
+        StatusCode::OK,
+        Json(json!({
+            "limit_per_second": config.limit_per_second,
+            "current_count": current_count,
+            "remaining": if config.limit_per_second > 0 {
+                (config.limit_per_second as i64).saturating_sub(current_count).max(0)
+            } else {
+                -1 // Unlimited
+            }
+        })),
+    )
+        .into_response()
+}
+
+/// Trigger graceful shutdown
+pub async fn trigger_graceful_shutdown(State(app_state): State<Arc<AppState>>) -> Response {
+    let handler = match &app_state.mesh_handler {
+        Some(h) => h.clone(),
+        None => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"error": "mesh not enabled"})),
+            )
+                .into_response();
+        }
+    };
+    info!("Graceful shutdown triggered via API");
+    tokio::spawn(async move {
+        if let Err(e) = handler.graceful_shutdown().await {
+            warn!("Error during graceful shutdown: {}", e);
+        }
+    });
+    (
+        StatusCode::ACCEPTED,
+        Json(json!({"status": "shutdown initiated"})),
+    )
+        .into_response()
+}

third_party/sglang/sgl-model-gateway/src/routers/mesh/mod.rs

@@ -0,0 +1,7 @@
+//! Mesh cluster management HTTP handlers
+//!
+//! This module provides HTTP API endpoints for mesh cluster management.
+
+mod handlers;
+
+pub use handlers::*;

third_party/sglang/sgl-model-gateway/src/routers/mod.rs

@@ -0,0 +1,206 @@
+//! Router implementations
+
+use std::fmt::Debug;
+
+use async_trait::async_trait;
+use axum::{
+    body::Body,
+    extract::Request,
+    http::{HeaderMap, StatusCode},
+    response::{IntoResponse, Response},
+};
+
+use crate::protocols::{
+    chat::ChatCompletionRequest,
+    classify::ClassifyRequest,
+    completion::CompletionRequest,
+    embedding::EmbeddingRequest,
+    generate::GenerateRequest,
+    rerank::RerankRequest,
+    responses::{ResponsesGetParams, ResponsesRequest},
+};
+
+pub mod conversations;
+pub mod error;
+pub mod factory;
+pub mod grpc;
+pub mod header_utils;
+pub mod http;
+pub mod mcp_utils;
+pub mod mesh;
+pub mod openai;
+pub mod parse;
+pub mod persistence_utils;
+pub mod router_manager;
+pub mod tokenize;
+
+pub use factory::RouterFactory;
+// Re-export HTTP routers for convenience
+pub use http::{pd_router, pd_types, router};
+
+/// Core trait for all router implementations
+///
+/// This trait provides a unified interface for routing requests,
+/// regardless of whether it's a regular router or PD router.
+#[async_trait]
+pub trait RouterTrait: Send + Sync + Debug {
+    /// Get a reference to self as Any for downcasting
+    fn as_any(&self) -> &dyn std::any::Any;
+
+    /// Route a health generate request
+    async fn health_generate(&self, _req: Request<Body>) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "Health generate not implemented",
+        )
+            .into_response()
+    }
+
+    /// Get server information
+    async fn get_server_info(&self, _req: Request<Body>) -> Response {
+        (StatusCode::NOT_IMPLEMENTED, "Server info not implemented").into_response()
+    }
+
+    /// Get available models
+    async fn get_models(&self, _req: Request<Body>) -> Response {
+        (StatusCode::NOT_IMPLEMENTED, "Get models not implemented").into_response()
+    }
+
+    /// Get model information
+    async fn get_model_info(&self, _req: Request<Body>) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "Get model info not implemented",
+        )
+            .into_response()
+    }
+
+    /// Route a generate request
+    async fn route_generate(
+        &self,
+        _headers: Option<&HeaderMap>,
+        _body: &GenerateRequest,
+        _model_id: Option<&str>,
+    ) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "Generate endpoint not implemented",
+        )
+            .into_response()
+    }
+
+    /// Route a chat completion request
+    async fn route_chat(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ChatCompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response;
+
+    /// Route a completion request
+    async fn route_completion(
+        &self,
+        _headers: Option<&HeaderMap>,
+        _body: &CompletionRequest,
+        _model_id: Option<&str>,
+    ) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "Completion endpoint not implemented",
+        )
+            .into_response()
+    }
+
+    /// Route a responses request
+    async fn route_responses(
+        &self,
+        _headers: Option<&HeaderMap>,
+        _body: &ResponsesRequest,
+        _model_id: Option<&str>,
+    ) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "Responses endpoint not implemented",
+        )
+            .into_response()
+    }
+
+    /// Retrieve a stored/background response by id
+    async fn get_response(
+        &self,
+        _headers: Option<&HeaderMap>,
+        _response_id: &str,
+        _params: &ResponsesGetParams,
+    ) -> Response {
+        (StatusCode::NOT_IMPLEMENTED, "Get response not implemented").into_response()
+    }
+
+    /// Cancel a background response by id
+    async fn cancel_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "Cancel response not implemented",
+        )
+            .into_response()
+    }
+
+    /// Delete a response by id
+    async fn delete_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "Responses delete endpoint not implemented",
+        )
+            .into_response()
+    }
+
+    /// List input items of a response by id
+    async fn list_response_input_items(
+        &self,
+        _headers: Option<&HeaderMap>,
+        _response_id: &str,
+    ) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "Responses list input items endpoint not implemented",
+        )
+            .into_response()
+    }
+
+    /// Route embedding requests (OpenAI-compatible /v1/embeddings)
+    async fn route_embeddings(
+        &self,
+        _headers: Option<&HeaderMap>,
+        _body: &EmbeddingRequest,
+        _model_id: Option<&str>,
+    ) -> Response {
+        (StatusCode::NOT_IMPLEMENTED, "Embeddings not implemented").into_response()
+    }
+
+    /// Route classification requests (OpenAI-compatible /v1/classify)
+    async fn route_classify(
+        &self,
+        _headers: Option<&HeaderMap>,
+        _body: &ClassifyRequest,
+        _model_id: Option<&str>,
+    ) -> Response {
+        (StatusCode::NOT_IMPLEMENTED, "Classify not implemented").into_response()
+    }
+
+    /// Route rerank requests
+    async fn route_rerank(
+        &self,
+        _headers: Option<&HeaderMap>,
+        _body: &RerankRequest,
+        _model_id: Option<&str>,
+    ) -> Response {
+        (StatusCode::NOT_IMPLEMENTED, "Rerank not implemented").into_response()
+    }
+
+    /// Get router type name
+    fn router_type(&self) -> &'static str;
+
+    /// Check if this is a PD router
+    fn is_pd_mode(&self) -> bool {
+        self.router_type() == "pd"
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/openai/context.rs

@@ -0,0 +1,244 @@
+//! Request context types for OpenAI router pipeline.
+
+use std::sync::Arc;
+
+use axum::http::HeaderMap;
+use data_connector::{ConversationItemStorage, ConversationStorage, ResponseStorage};
+use serde_json::Value;
+use smg_mcp::McpManager;
+
+use super::provider::Provider;
+use crate::{
+    core::Worker,
+    protocols::{chat::ChatCompletionRequest, responses::ResponsesRequest},
+};
+
+pub struct RequestContext {
+    pub input: RequestInput,
+    pub components: ComponentRefs,
+    pub state: ProcessingState,
+}
+
+pub struct RequestInput {
+    pub request_type: RequestType,
+    pub headers: Option<HeaderMap>,
+    #[allow(dead_code)]
+    pub model_id: Option<String>,
+}
+
+pub enum RequestType {
+    Chat(Arc<ChatCompletionRequest>),
+    Responses(Arc<ResponsesRequest>),
+}
+
+#[derive(Clone)]
+pub struct SharedComponents {
+    pub client: reqwest::Client,
+}
+
+pub struct ResponsesComponents {
+    pub shared: SharedComponents,
+    pub mcp_manager: Arc<McpManager>,
+    pub response_storage: Arc<dyn ResponseStorage>,
+    pub conversation_storage: Arc<dyn ConversationStorage>,
+    pub conversation_item_storage: Arc<dyn ConversationItemStorage>,
+}
+
+pub enum ComponentRefs {
+    Shared(Arc<SharedComponents>),
+    Responses(Arc<ResponsesComponents>),
+}
+
+impl ComponentRefs {
+    pub fn client(&self) -> &reqwest::Client {
+        match self {
+            ComponentRefs::Shared(s) => &s.client,
+            ComponentRefs::Responses(r) => &r.shared.client,
+        }
+    }
+
+    pub fn mcp_manager(&self) -> Option<&Arc<McpManager>> {
+        match self {
+            ComponentRefs::Shared(_) => None,
+            ComponentRefs::Responses(r) => Some(&r.mcp_manager),
+        }
+    }
+
+    pub fn response_storage(&self) -> Option<&Arc<dyn ResponseStorage>> {
+        match self {
+            ComponentRefs::Shared(_) => None,
+            ComponentRefs::Responses(r) => Some(&r.response_storage),
+        }
+    }
+
+    pub fn conversation_storage(&self) -> Option<&Arc<dyn ConversationStorage>> {
+        match self {
+            ComponentRefs::Shared(_) => None,
+            ComponentRefs::Responses(r) => Some(&r.conversation_storage),
+        }
+    }
+
+    pub fn conversation_item_storage(&self) -> Option<&Arc<dyn ConversationItemStorage>> {
+        match self {
+            ComponentRefs::Shared(_) => None,
+            ComponentRefs::Responses(r) => Some(&r.conversation_item_storage),
+        }
+    }
+}
+
+#[derive(Default)]
+pub struct ProcessingState {
+    pub worker: Option<WorkerSelection>,
+    pub payload: Option<PayloadState>,
+}
+
+pub struct WorkerSelection {
+    pub worker: Arc<dyn Worker>,
+    #[allow(dead_code)]
+    pub provider: Arc<dyn Provider>,
+}
+
+pub struct PayloadState {
+    pub json: Value,
+    pub url: String,
+    pub previous_response_id: Option<String>,
+}
+
+impl RequestContext {
+    pub fn for_responses(
+        request: Arc<ResponsesRequest>,
+        headers: Option<HeaderMap>,
+        model_id: Option<String>,
+        components: ComponentRefs,
+    ) -> Self {
+        Self {
+            input: RequestInput {
+                request_type: RequestType::Responses(request),
+                headers,
+                model_id,
+            },
+            components,
+            state: ProcessingState::default(),
+        }
+    }
+
+    pub fn for_chat(
+        request: Arc<ChatCompletionRequest>,
+        headers: Option<HeaderMap>,
+        model_id: Option<String>,
+        components: ComponentRefs,
+    ) -> Self {
+        Self {
+            input: RequestInput {
+                request_type: RequestType::Chat(request),
+                headers,
+                model_id,
+            },
+            components,
+            state: ProcessingState::default(),
+        }
+    }
+}
+
+impl RequestContext {
+    pub fn responses_request(&self) -> &ResponsesRequest {
+        match &self.input.request_type {
+            RequestType::Responses(req) => req.as_ref(),
+            _ => panic!("Expected responses request"),
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn responses_request_arc(&self) -> Arc<ResponsesRequest> {
+        match &self.input.request_type {
+            RequestType::Responses(req) => Arc::clone(req),
+            _ => panic!("Expected responses request"),
+        }
+    }
+
+    pub fn is_streaming(&self) -> bool {
+        match &self.input.request_type {
+            RequestType::Chat(req) => req.stream,
+            RequestType::Responses(req) => req.stream.unwrap_or(false),
+        }
+    }
+
+    pub fn headers(&self) -> Option<&HeaderMap> {
+        self.input.headers.as_ref()
+    }
+
+    #[allow(dead_code)]
+    pub fn model_id(&self) -> Option<&str> {
+        self.input.model_id.as_deref()
+    }
+
+    pub fn worker(&self) -> Option<&Arc<dyn Worker>> {
+        self.state.worker.as_ref().map(|w| &w.worker)
+    }
+
+    #[allow(dead_code)]
+    pub fn provider(&self) -> Option<&dyn Provider> {
+        self.state.worker.as_ref().map(|w| w.provider.as_ref())
+    }
+
+    pub fn payload(&self) -> Option<&PayloadState> {
+        self.state.payload.as_ref()
+    }
+
+    pub fn take_payload(&mut self) -> Option<PayloadState> {
+        self.state.payload.take()
+    }
+}
+
+pub struct StorageHandles {
+    pub response: Arc<dyn ResponseStorage>,
+    pub conversation: Arc<dyn ConversationStorage>,
+    pub conversation_item: Arc<dyn ConversationItemStorage>,
+}
+
+pub struct OwnedStreamingContext {
+    pub url: String,
+    pub payload: Value,
+    pub original_body: ResponsesRequest,
+    pub previous_response_id: Option<String>,
+    pub storage: StorageHandles,
+}
+
+impl RequestContext {
+    pub fn into_streaming_context(mut self) -> OwnedStreamingContext {
+        let payload_state = self.take_payload().expect("Payload not prepared");
+
+        OwnedStreamingContext {
+            url: payload_state.url,
+            payload: payload_state.json,
+            original_body: self.responses_request().clone(),
+            previous_response_id: payload_state.previous_response_id,
+            storage: StorageHandles {
+                response: self
+                    .components
+                    .response_storage()
+                    .expect("Response storage required")
+                    .clone(),
+                conversation: self
+                    .components
+                    .conversation_storage()
+                    .expect("Conversation storage required")
+                    .clone(),
+                conversation_item: self
+                    .components
+                    .conversation_item_storage()
+                    .expect("Conversation item storage required")
+                    .clone(),
+            },
+        }
+    }
+}
+
+pub struct StreamingEventContext<'a> {
+    pub server_label: &'a str,
+    pub original_request: &'a ResponsesRequest,
+    pub previous_response_id: Option<&'a str>,
+    pub server_keys: &'a [String],
+}
+
+pub type StreamingRequest = OwnedStreamingContext;

third_party/sglang/sgl-model-gateway/src/routers/openai/mod.rs

@@ -0,0 +1,15 @@
+//! OpenAI-compatible router implementation
+//!
+//! This module provides OpenAI-compatible API routing with support for:
+//! - Streaming and non-streaming responses
+//! - MCP (Model Context Protocol) tool calling
+//! - Response storage and conversation management
+//! - Multi-turn tool execution loops
+//! - SSE (Server-Sent Events) streaming
+
+mod context;
+mod provider;
+pub mod responses;
+mod router;
+
+pub use router::OpenAIRouter;

third_party/sglang/sgl-model-gateway/src/routers/openai/provider.rs

@@ -0,0 +1,252 @@
+//! Provider abstractions for vendor-specific API transformations.
+
+use std::{collections::HashMap, sync::Arc};
+
+use reqwest::RequestBuilder;
+use serde_json::Value;
+use thiserror::Error;
+
+use crate::core::{model_type::Endpoint, ProviderType};
+
+const SGLANG_FIELDS: &[&str] = &[
+    "request_id",
+    "priority",
+    "top_k",
+    "min_p",
+    "min_tokens",
+    "regex",
+    "ebnf",
+    "json_schema",
+    "stop_token_ids",
+    "no_stop_trim",
+    "ignore_eos",
+    "continue_final_message",
+    "skip_special_tokens",
+    "lora_path",
+    "session_params",
+    "separate_reasoning",
+    "stream_reasoning",
+    "chat_template",
+    "chat_template_kwargs",
+    "return_hidden_states",
+    "repetition_penalty",
+    "sampling_seed",
+    "backend_url",
+];
+
+fn strip_sglang_fields(payload: &mut Value) {
+    if let Some(obj) = payload.as_object_mut() {
+        for field in SGLANG_FIELDS {
+            obj.remove(*field);
+        }
+    }
+}
+
+#[derive(Error, Debug)]
+pub enum ProviderError {
+    #[error("Unsupported endpoint: {0:?}")]
+    UnsupportedEndpoint(Endpoint),
+
+    #[error("Transform error: {0}")]
+    TransformError(String),
+}
+
+/// Default `transform_request` strips SGLang fields.
+pub trait Provider: Send + Sync {
+    fn provider_type(&self) -> ProviderType;
+
+    fn transform_request(
+        &self,
+        payload: &mut Value,
+        _endpoint: Endpoint,
+    ) -> Result<(), ProviderError> {
+        strip_sglang_fields(payload);
+        Ok(())
+    }
+
+    fn transform_response(
+        &self,
+        _response: &mut Value,
+        _endpoint: Endpoint,
+    ) -> Result<(), ProviderError> {
+        Ok(())
+    }
+
+    fn apply_headers(&self, builder: RequestBuilder) -> RequestBuilder {
+        builder
+    }
+}
+
+pub struct SGLangProvider;
+
+impl Provider for SGLangProvider {
+    fn provider_type(&self) -> ProviderType {
+        ProviderType::OpenAI
+    }
+
+    fn transform_request(
+        &self,
+        _payload: &mut Value,
+        _endpoint: Endpoint,
+    ) -> Result<(), ProviderError> {
+        Ok(())
+    }
+}
+
+pub struct OpenAIProvider;
+
+impl Provider for OpenAIProvider {
+    fn provider_type(&self) -> ProviderType {
+        ProviderType::OpenAI
+    }
+}
+
+pub struct AnthropicProvider;
+
+impl Provider for AnthropicProvider {
+    fn provider_type(&self) -> ProviderType {
+        ProviderType::Anthropic
+    }
+}
+
+pub struct XAIProvider;
+
+impl Provider for XAIProvider {
+    fn provider_type(&self) -> ProviderType {
+        ProviderType::XAI
+    }
+
+    fn transform_request(
+        &self,
+        payload: &mut Value,
+        endpoint: Endpoint,
+    ) -> Result<(), ProviderError> {
+        strip_sglang_fields(payload);
+
+        if endpoint == Endpoint::Responses {
+            if let Some(obj) = payload.as_object_mut() {
+                Self::transform_responses_input(obj);
+            }
+        }
+        Ok(())
+    }
+}
+
+impl XAIProvider {
+    fn transform_responses_input(obj: &mut serde_json::Map<String, Value>) {
+        let Some(input_arr) = obj.get_mut("input").and_then(Value::as_array_mut) else {
+            return;
+        };
+
+        for item in input_arr.iter_mut().filter_map(Value::as_object_mut) {
+            item.remove("id");
+            item.remove("status");
+
+            let Some(content_arr) = item.get_mut("content").and_then(Value::as_array_mut) else {
+                continue;
+            };
+
+            for content in content_arr.iter_mut().filter_map(Value::as_object_mut) {
+                if content.get("type").and_then(Value::as_str) == Some("output_text") {
+                    content.insert("type".to_string(), Value::String("input_text".to_string()));
+                }
+            }
+        }
+    }
+}
+
+pub struct GeminiProvider;
+
+impl Provider for GeminiProvider {
+    fn provider_type(&self) -> ProviderType {
+        ProviderType::Gemini
+    }
+
+    fn transform_request(
+        &self,
+        payload: &mut Value,
+        endpoint: Endpoint,
+    ) -> Result<(), ProviderError> {
+        strip_sglang_fields(payload);
+
+        if endpoint == Endpoint::Chat {
+            if let Some(obj) = payload.as_object_mut() {
+                if obj.get("logprobs").and_then(|v| v.as_bool()) == Some(false) {
+                    obj.remove("logprobs");
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+pub struct ProviderRegistry {
+    providers: HashMap<ProviderType, Arc<dyn Provider>>,
+    default_provider: Arc<dyn Provider>,
+}
+
+impl Default for ProviderRegistry {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ProviderRegistry {
+    pub fn new() -> Self {
+        let mut providers = HashMap::new();
+
+        providers.insert(
+            ProviderType::OpenAI,
+            Arc::new(OpenAIProvider) as Arc<dyn Provider>,
+        );
+        providers.insert(
+            ProviderType::XAI,
+            Arc::new(XAIProvider) as Arc<dyn Provider>,
+        );
+        providers.insert(
+            ProviderType::Gemini,
+            Arc::new(GeminiProvider) as Arc<dyn Provider>,
+        );
+        providers.insert(
+            ProviderType::Anthropic,
+            Arc::new(AnthropicProvider) as Arc<dyn Provider>,
+        );
+
+        Self {
+            providers,
+            default_provider: Arc::new(SGLangProvider),
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn get(&self, provider_type: &ProviderType) -> &dyn Provider {
+        self.providers
+            .get(provider_type)
+            .map(|p| p.as_ref())
+            .unwrap_or(self.default_provider.as_ref())
+    }
+
+    pub fn get_arc(&self, provider_type: &ProviderType) -> Arc<dyn Provider> {
+        self.providers
+            .get(provider_type)
+            .cloned()
+            .unwrap_or_else(|| Arc::clone(&self.default_provider))
+    }
+
+    #[allow(dead_code)]
+    pub fn get_for_model(&self, model_name: &str) -> &dyn Provider {
+        match ProviderType::from_model_name(model_name) {
+            Some(pt) => self.get(&pt),
+            None => self.default_provider.as_ref(),
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn default_provider(&self) -> &dyn Provider {
+        self.default_provider.as_ref()
+    }
+
+    pub fn default_provider_arc(&self) -> Arc<dyn Provider> {
+        Arc::clone(&self.default_provider)
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/openai/responses/accumulator.rs

@@ -0,0 +1,164 @@
+//! Streaming response accumulator for persisting responses.
+
+use serde_json::Value;
+use tracing::warn;
+
+use super::common::{extract_output_index, get_event_type};
+use crate::protocols::event_types::{OutputItemEvent, ResponseEvent};
+
+// ============================================================================
+// Streaming Response Accumulator
+// ============================================================================
+
+/// Helper that parses SSE frames from the OpenAI responses stream and
+/// accumulates enough information to persist the final response locally.
+pub(super) struct StreamingResponseAccumulator {
+    /// The initial `response.created` payload (if emitted).
+    initial_response: Option<Value>,
+    /// The final `response.completed` payload (if emitted).
+    completed_response: Option<Value>,
+    /// Collected output items keyed by the upstream output index, used when
+    /// a final response payload is absent and we need to synthesize one.
+    output_items: Vec<(usize, Value)>,
+    /// Captured error payload (if the upstream stream fails midway).
+    encountered_error: Option<Value>,
+}
+
+impl StreamingResponseAccumulator {
+    pub fn new() -> Self {
+        Self {
+            initial_response: None,
+            completed_response: None,
+            output_items: Vec::new(),
+            encountered_error: None,
+        }
+    }
+
+    /// Feed the accumulator with the next SSE chunk.
+    pub fn ingest_block(&mut self, block: &str) {
+        if block.trim().is_empty() {
+            return;
+        }
+        self.process_block(block);
+    }
+
+    /// Consume the accumulator and produce the best-effort final response value.
+    pub fn into_final_response(mut self) -> Option<Value> {
+        if self.completed_response.is_some() {
+            return self.completed_response;
+        }
+
+        self.build_fallback_response()
+    }
+
+    pub fn encountered_error(&self) -> Option<&Value> {
+        self.encountered_error.as_ref()
+    }
+
+    pub fn original_response_id(&self) -> Option<&str> {
+        self.initial_response
+            .as_ref()
+            .and_then(|response| response.get("id"))
+            .and_then(|id| id.as_str())
+    }
+
+    pub fn snapshot_final_response(&self) -> Option<Value> {
+        if let Some(resp) = &self.completed_response {
+            return Some(resp.clone());
+        }
+        self.build_fallback_response_snapshot()
+    }
+
+    fn build_fallback_response_snapshot(&self) -> Option<Value> {
+        let mut response = self.initial_response.clone()?;
+
+        if let Some(obj) = response.as_object_mut() {
+            obj.insert("status".to_string(), Value::String("completed".to_string()));
+
+            let mut output_items = self.output_items.clone();
+            output_items.sort_by_key(|(index, _)| *index);
+            let outputs: Vec<Value> = output_items.into_iter().map(|(_, item)| item).collect();
+            obj.insert("output".to_string(), Value::Array(outputs));
+        }
+
+        Some(response)
+    }
+
+    fn process_block(&mut self, block: &str) {
+        let trimmed = block.trim();
+        if trimmed.is_empty() {
+            return;
+        }
+
+        let mut event_name: Option<String> = None;
+        let mut data_lines: Vec<String> = Vec::new();
+
+        for line in trimmed.lines() {
+            if let Some(rest) = line.strip_prefix("event:") {
+                event_name = Some(rest.trim().to_string());
+            } else if let Some(rest) = line.strip_prefix("data:") {
+                data_lines.push(rest.trim_start().to_string());
+            }
+        }
+
+        let data_payload = data_lines.join("\n");
+        if data_payload.is_empty() {
+            return;
+        }
+
+        self.handle_event(event_name.as_deref(), &data_payload);
+    }
+
+    fn handle_event(&mut self, event_name: Option<&str>, data_payload: &str) {
+        let parsed: Value = match serde_json::from_str(data_payload) {
+            Ok(value) => value,
+            Err(err) => {
+                warn!("Failed to parse streaming event JSON: {}", err);
+                return;
+            }
+        };
+
+        match get_event_type(event_name, &parsed) {
+            ResponseEvent::CREATED => {
+                if self.initial_response.is_none() {
+                    if let Some(response) = parsed.get("response") {
+                        self.initial_response = Some(response.clone());
+                    }
+                }
+            }
+            ResponseEvent::COMPLETED => {
+                if let Some(response) = parsed.get("response") {
+                    self.completed_response = Some(response.clone());
+                }
+            }
+            OutputItemEvent::DONE => {
+                if let (Some(index), Some(item)) =
+                    (extract_output_index(&parsed), parsed.get("item"))
+                {
+                    self.output_items.push((index, item.clone()));
+                }
+            }
+            "response.error" => {
+                self.encountered_error = Some(parsed);
+            }
+            _ => {}
+        }
+    }
+
+    fn build_fallback_response(&mut self) -> Option<Value> {
+        let mut response = self.initial_response.clone()?;
+
+        if let Some(obj) = response.as_object_mut() {
+            obj.insert("status".to_string(), Value::String("completed".to_string()));
+
+            self.output_items.sort_by_key(|(index, _)| *index);
+            let outputs: Vec<Value> = std::mem::take(&mut self.output_items)
+                .into_iter()
+                .map(|(_, item)| item)
+                .collect();
+            obj.insert("output".to_string(), Value::Array(outputs));
+        }
+
+        Some(response)
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/openai/responses/common.rs

@@ -0,0 +1,113 @@
+//! Common SSE parsing and processing utilities for OpenAI responses
+//!
+//! This module contains shared helpers used by both streaming and accumulator modules.
+
+use std::borrow::Cow;
+
+use serde_json::Value;
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/// Extract output_index from a JSON value
+#[inline]
+pub(super) fn extract_output_index(value: &Value) -> Option<usize> {
+    value.get("output_index")?.as_u64().map(|v| v as usize)
+}
+
+/// Get event type from event name or parsed JSON, returning a reference to avoid allocation
+#[inline]
+pub(super) fn get_event_type<'a>(event_name: Option<&'a str>, parsed: &'a Value) -> &'a str {
+    event_name
+        .or_else(|| parsed.get("type").and_then(|v| v.as_str()))
+        .unwrap_or("")
+}
+
+// ============================================================================
+// Chunk Processor
+// ============================================================================
+
+/// Processes incoming byte chunks into complete SSE blocks.
+/// Handles buffering of partial chunks and CRLF normalization.
+pub(super) struct ChunkProcessor {
+    pending: String,
+}
+
+impl ChunkProcessor {
+    pub fn new() -> Self {
+        Self {
+            pending: String::new(),
+        }
+    }
+
+    /// Append a chunk to the buffer, normalizing line endings
+    pub fn push_chunk(&mut self, chunk: &[u8]) {
+        let chunk_str = match std::str::from_utf8(chunk) {
+            Ok(s) => Cow::Borrowed(s),
+            Err(_) => Cow::Owned(String::from_utf8_lossy(chunk).into_owned()),
+        };
+        // Normalize CRLF to LF without extra allocation
+        let mut chars = chunk_str.chars().peekable();
+        while let Some(c) = chars.next() {
+            if c == '\r' && chars.peek() == Some(&'\n') {
+                // Skip \r when followed by \n
+                continue;
+            }
+            self.pending.push(c);
+        }
+    }
+
+    /// Extract the next complete SSE block from the buffer, if available
+    pub fn next_block(&mut self) -> Option<String> {
+        loop {
+            let pos = self.pending.find("\n\n")?;
+            let block = self.pending[..pos].to_string();
+            self.pending.drain(..pos + 2);
+
+            if !block.trim().is_empty() {
+                return Some(block);
+            }
+            // If block is empty, loop again to find the next one
+        }
+    }
+
+    /// Check if there's remaining content in the buffer
+    pub fn has_remaining(&self) -> bool {
+        !self.pending.trim().is_empty()
+    }
+
+    /// Take any remaining content from the buffer
+    pub fn take_remaining(&mut self) -> String {
+        std::mem::take(&mut self.pending)
+    }
+}
+
+// ============================================================================
+// SSE Parsing
+// ============================================================================
+
+/// Parse an SSE block into event name and data
+///
+/// Returns borrowed strings when possible to avoid allocations in hot paths.
+/// Only allocates when multiple data lines need to be joined.
+pub(super) fn parse_sse_block(block: &str) -> (Option<&str>, Cow<'_, str>) {
+    let mut event_name: Option<&str> = None;
+    let mut data_lines: Vec<&str> = Vec::new();
+
+    for line in block.lines() {
+        if let Some(rest) = line.strip_prefix("event:") {
+            event_name = Some(rest.trim());
+        } else if let Some(rest) = line.strip_prefix("data:") {
+            data_lines.push(rest.trim_start());
+        }
+    }
+
+    let data = if data_lines.len() == 1 {
+        Cow::Borrowed(data_lines[0])
+    } else {
+        Cow::Owned(data_lines.join("\n"))
+    };
+
+    (event_name, data)
+}

third_party/sglang/sgl-model-gateway/src/routers/openai/responses/mcp.rs

@@ -0,0 +1,893 @@
+//! MCP (Model Context Protocol) Integration Module
+//!
+//! This module contains all MCP-related functionality for the OpenAI router:
+//! - Tool loop state management for multi-turn tool calling
+//! - MCP tool execution and result handling
+//! - Output item builders for MCP-specific response formats
+//! - SSE event generation for streaming MCP operations
+//! - Payload transformation for MCP tool interception
+//! - Metadata injection for MCP operations
+
+use std::{io, sync::Arc};
+
+use axum::http::HeaderMap;
+use bytes::Bytes;
+use serde_json::{json, to_value, Value};
+use smg_mcp as mcp;
+use tokio::sync::mpsc;
+use tracing::{debug, info, warn};
+
+use crate::{
+    protocols::{
+        event_types::{is_function_call_type, ItemType, McpEvent, OutputItemEvent},
+        responses::{generate_id, ResponseInput, ResponsesRequest},
+    },
+    routers::{
+        header_utils::apply_request_headers,
+        mcp_utils::{extract_server_label, McpLoopConfig},
+    },
+};
+
+// ============================================================================
+// Configuration and State Types
+// ============================================================================
+
+/// State for tracking multi-turn tool calling loop
+pub(super) struct ToolLoopState {
+    /// Current iteration number (starts at 0, increments with each tool call)
+    pub iteration: usize,
+    /// Total number of tool calls executed
+    pub total_calls: usize,
+    /// Conversation history (function_call and function_call_output items)
+    pub conversation_history: Vec<Value>,
+    /// Original user input (preserved for building resume payloads)
+    pub original_input: ResponseInput,
+}
+
+impl ToolLoopState {
+    pub fn new(original_input: ResponseInput) -> Self {
+        Self {
+            iteration: 0,
+            total_calls: 0,
+            conversation_history: Vec::new(),
+            original_input,
+        }
+    }
+
+    /// Record a tool call in the loop state
+    pub fn record_call(
+        &mut self,
+        call_id: String,
+        tool_name: String,
+        args_json_str: String,
+        output_str: String,
+    ) {
+        // Add function_call item to history
+        let func_item = json!({
+            "type": ItemType::FUNCTION_CALL,
+            "call_id": call_id,
+            "name": tool_name,
+            "arguments": args_json_str
+        });
+        self.conversation_history.push(func_item);
+
+        // Add function_call_output item to history
+        let output_item = json!({
+            "type": "function_call_output",
+            "call_id": call_id,
+            "output": output_str
+        });
+        self.conversation_history.push(output_item);
+    }
+}
+
+/// Represents a function call being accumulated across delta events
+#[derive(Debug, Clone)]
+pub(super) struct FunctionCallInProgress {
+    pub call_id: String,
+    pub name: String,
+    pub arguments_buffer: String,
+    pub output_index: usize,
+    pub last_obfuscation: Option<String>,
+    pub assigned_output_index: Option<usize>,
+}
+
+impl FunctionCallInProgress {
+    pub fn new(call_id: String, output_index: usize) -> Self {
+        Self {
+            call_id,
+            name: String::new(),
+            arguments_buffer: String::new(),
+            output_index,
+            last_obfuscation: None,
+            assigned_output_index: None,
+        }
+    }
+
+    pub fn is_complete(&self) -> bool {
+        // A tool call is complete if it has a name
+        !self.name.is_empty()
+    }
+
+    pub fn effective_output_index(&self) -> usize {
+        self.assigned_output_index.unwrap_or(self.output_index)
+    }
+}
+
+// ============================================================================
+// Tool Execution
+// ============================================================================
+
+/// Execute detected tool calls and send completion events to client
+/// Returns false if client disconnected during execution
+pub(super) async fn execute_streaming_tool_calls(
+    pending_calls: Vec<FunctionCallInProgress>,
+    active_mcp: &Arc<mcp::McpManager>,
+    tx: &mpsc::UnboundedSender<Result<Bytes, io::Error>>,
+    state: &mut ToolLoopState,
+    server_label: &str,
+    sequence_number: &mut u64,
+) -> bool {
+    // Execute all pending tool calls (sequential, as PR3 is skipped)
+    for call in pending_calls {
+        // Skip if name is empty (invalid call)
+        if call.name.is_empty() {
+            warn!(
+                "Skipping incomplete tool call: name is empty, args_len={}",
+                call.arguments_buffer.len()
+            );
+            continue;
+        }
+
+        info!(
+            "Executing tool call during streaming: {} ({})",
+            call.name, call.call_id
+        );
+
+        // Use empty JSON object if arguments_buffer is empty
+        let args_str = if call.arguments_buffer.is_empty() {
+            "{}"
+        } else {
+            &call.arguments_buffer
+        };
+
+        // Call tool directly - manager handles parsing and type coercion
+        debug!("Calling MCP tool '{}' with args: {}", call.name, args_str);
+        let call_result = active_mcp.call_tool(&call.name, args_str).await;
+        let (output_str, success, error_msg) = match call_result {
+            Ok(result) => match serde_json::to_string(&result) {
+                Ok(output) => (output, true, None),
+                Err(e) => {
+                    let err = format!("Failed to serialize tool result: {}", e);
+                    warn!("{}", err);
+                    (json!({ "error": &err }).to_string(), false, Some(err))
+                }
+            },
+            Err(err) => {
+                let err_str = format!("tool call failed: {}", err);
+                warn!("Tool execution failed during streaming: {}", err_str);
+                (
+                    json!({ "error": &err_str }).to_string(),
+                    false,
+                    Some(err_str),
+                )
+            }
+        };
+
+        // Send mcp_call completion event to client
+        if !send_mcp_call_completion_events_with_error(
+            tx,
+            &call,
+            &output_str,
+            server_label,
+            success,
+            error_msg.as_deref(),
+            sequence_number,
+        ) {
+            // Client disconnected, no point continuing tool execution
+            return false;
+        }
+
+        // Record the call
+        state.record_call(call.call_id, call.name, call.arguments_buffer, output_str);
+    }
+    true
+}
+
+// ============================================================================
+// Payload Transformation
+// ============================================================================
+
+/// Transform payload to replace MCP tools with function tools
+pub(super) fn prepare_mcp_tools_as_functions(
+    payload: &mut Value,
+    active_mcp: &Arc<mcp::McpManager>,
+    server_keys: &[String],
+) {
+    if let Some(obj) = payload.as_object_mut() {
+        // Remove any non-function tools from outgoing payload
+        if let Some(v) = obj.get_mut("tools") {
+            if let Some(arr) = v.as_array_mut() {
+                arr.retain(|item| {
+                    item.get("type")
+                        .and_then(|v| v.as_str())
+                        .map(|s| s == ItemType::FUNCTION)
+                        .unwrap_or(false)
+                });
+            }
+        }
+
+        // Build function tools for all discovered MCP tools
+        let tools = active_mcp.list_tools_for_servers(server_keys);
+        let mut tools_json = Vec::with_capacity(tools.len());
+        for t in tools {
+            let parameters = Value::Object((*t.input_schema).clone());
+            let tool = serde_json::json!({
+                "type": ItemType::FUNCTION,
+                "name": t.name,
+                "description": t.description,
+                "parameters": parameters
+            });
+            tools_json.push(tool);
+        }
+        if !tools_json.is_empty() {
+            obj.insert("tools".to_string(), Value::Array(tools_json));
+            obj.insert("tool_choice".to_string(), Value::String("auto".to_string()));
+        }
+    }
+}
+
+/// Build a resume payload with conversation history
+pub(super) fn build_resume_payload(
+    base_payload: &Value,
+    conversation_history: &[Value],
+    original_input: &ResponseInput,
+    tools_json: &Value,
+    is_streaming: bool,
+) -> Result<Value, String> {
+    // Clone the base payload which already has cleaned fields
+    let mut payload = base_payload.clone();
+
+    let obj = payload
+        .as_object_mut()
+        .ok_or_else(|| "payload not an object".to_string())?;
+
+    // Build input array: start with original user input
+    // Pre-allocate: 1 for user message + conversation history
+    let mut input_array = Vec::with_capacity(1 + conversation_history.len());
+
+    // Add original user message
+    // For structured input, serialize the original input items
+    match original_input {
+        ResponseInput::Text(text) => {
+            let user_item = json!({
+                "type": "message",
+                "role": "user",
+                "content": [{ "type": "input_text", "text": text }]
+            });
+            input_array.push(user_item);
+        }
+        ResponseInput::Items(items) => {
+            // Items are ResponseInputOutputItem (including SimpleInputMessage), convert to JSON
+            if let Ok(items_value) = to_value(items) {
+                if let Some(items_arr) = items_value.as_array() {
+                    input_array.extend_from_slice(items_arr);
+                }
+            }
+        }
+    }
+
+    // Add all conversation history (function calls and outputs)
+    input_array.extend_from_slice(conversation_history);
+
+    obj.insert("input".to_string(), Value::Array(input_array));
+
+    // Use the transformed tools (function tools, not MCP tools)
+    if let Some(tools_arr) = tools_json.as_array() {
+        if !tools_arr.is_empty() {
+            obj.insert("tools".to_string(), tools_json.clone());
+        }
+    }
+
+    // Set streaming mode based on caller's context
+    obj.insert("stream".to_string(), Value::Bool(is_streaming));
+    obj.insert("store".to_string(), Value::Bool(false));
+
+    // Note: SGLang-specific fields were already removed from base_payload
+    // before it was passed to execute_tool_loop (see route_responses lines 1935-1946)
+
+    Ok(payload)
+}
+
+// ============================================================================
+// SSE Event Senders
+// ============================================================================
+
+/// Send mcp_list_tools events to client at the start of streaming
+/// Returns false if client disconnected
+pub(super) fn send_mcp_list_tools_events(
+    tx: &mpsc::UnboundedSender<Result<Bytes, io::Error>>,
+    mcp: &Arc<mcp::McpManager>,
+    server_label: &str,
+    output_index: usize,
+    sequence_number: &mut u64,
+    server_keys: &[String],
+) -> bool {
+    let tools_item_full = build_mcp_list_tools_item(mcp, server_label, server_keys);
+    let item_id = tools_item_full
+        .get("id")
+        .and_then(|v| v.as_str())
+        .unwrap_or("");
+
+    // Create empty tools version for the initial added event
+    let mut tools_item_empty = tools_item_full.clone();
+    if let Some(obj) = tools_item_empty.as_object_mut() {
+        obj.insert("tools".to_string(), json!([]));
+    }
+
+    // Event 1: response.output_item.added with empty tools
+    let event1_payload = json!({
+        "type": OutputItemEvent::ADDED,
+        "sequence_number": *sequence_number,
+        "output_index": output_index,
+        "item": tools_item_empty
+    });
+    *sequence_number += 1;
+    let event1 = format!(
+        "event: {}\ndata: {}\n\n",
+        OutputItemEvent::ADDED,
+        event1_payload
+    );
+    if tx.send(Ok(Bytes::from(event1))).is_err() {
+        return false; // Client disconnected
+    }
+
+    // Event 2: response.mcp_list_tools.in_progress
+    let event2_payload = json!({
+        "type": McpEvent::LIST_TOOLS_IN_PROGRESS,
+        "sequence_number": *sequence_number,
+        "output_index": output_index,
+        "item_id": item_id
+    });
+    *sequence_number += 1;
+    let event2 = format!(
+        "event: {}\ndata: {}\n\n",
+        McpEvent::LIST_TOOLS_IN_PROGRESS,
+        event2_payload
+    );
+    if tx.send(Ok(Bytes::from(event2))).is_err() {
+        return false;
+    }
+
+    // Event 3: response.mcp_list_tools.completed
+    let event3_payload = json!({
+        "type": McpEvent::LIST_TOOLS_COMPLETED,
+        "sequence_number": *sequence_number,
+        "output_index": output_index,
+        "item_id": item_id
+    });
+    *sequence_number += 1;
+    let event3 = format!(
+        "event: {}\ndata: {}\n\n",
+        McpEvent::LIST_TOOLS_COMPLETED,
+        event3_payload
+    );
+    if tx.send(Ok(Bytes::from(event3))).is_err() {
+        return false;
+    }
+
+    // Event 4: response.output_item.done with full tools list
+    let event4_payload = json!({
+        "type": OutputItemEvent::DONE,
+        "sequence_number": *sequence_number,
+        "output_index": output_index,
+        "item": tools_item_full
+    });
+    *sequence_number += 1;
+    let event4 = format!(
+        "event: {}\ndata: {}\n\n",
+        OutputItemEvent::DONE,
+        event4_payload
+    );
+    tx.send(Ok(Bytes::from(event4))).is_ok()
+}
+
+/// Send mcp_call completion events after tool execution
+/// Returns false if client disconnected
+pub(super) fn send_mcp_call_completion_events_with_error(
+    tx: &mpsc::UnboundedSender<Result<Bytes, io::Error>>,
+    call: &FunctionCallInProgress,
+    output: &str,
+    server_label: &str,
+    success: bool,
+    error_msg: Option<&str>,
+    sequence_number: &mut u64,
+) -> bool {
+    let effective_output_index = call.effective_output_index();
+
+    // Build mcp_call item (reuse existing function)
+    let mcp_call_item = build_mcp_call_item(
+        &call.name,
+        &call.arguments_buffer,
+        output,
+        server_label,
+        success,
+        error_msg,
+    );
+
+    // Get the mcp_call item_id
+    let item_id = mcp_call_item
+        .get("id")
+        .and_then(|v| v.as_str())
+        .unwrap_or("");
+
+    // Event 1: response.mcp_call.completed
+    let completed_payload = json!({
+        "type": McpEvent::CALL_COMPLETED,
+        "sequence_number": *sequence_number,
+        "output_index": effective_output_index,
+        "item_id": item_id
+    });
+    *sequence_number += 1;
+
+    let completed_event = format!(
+        "event: {}\ndata: {}\n\n",
+        McpEvent::CALL_COMPLETED,
+        completed_payload
+    );
+    if tx.send(Ok(Bytes::from(completed_event))).is_err() {
+        return false;
+    }
+
+    // Event 2: response.output_item.done (with completed mcp_call)
+    let done_payload = json!({
+        "type": OutputItemEvent::DONE,
+        "sequence_number": *sequence_number,
+        "output_index": effective_output_index,
+        "item": mcp_call_item
+    });
+    *sequence_number += 1;
+
+    let done_event = format!(
+        "event: {}\ndata: {}\n\n",
+        OutputItemEvent::DONE,
+        done_payload
+    );
+    tx.send(Ok(Bytes::from(done_event))).is_ok()
+}
+
+// ============================================================================
+// Metadata Injection
+// ============================================================================
+
+/// Inject MCP metadata into a streaming response
+pub(super) fn inject_mcp_metadata_streaming(
+    response: &mut Value,
+    state: &ToolLoopState,
+    mcp: &Arc<mcp::McpManager>,
+    server_label: &str,
+    server_keys: &[String],
+) {
+    if let Some(output_array) = response.get_mut("output").and_then(|v| v.as_array_mut()) {
+        output_array.retain(|item| {
+            item.get("type").and_then(|t| t.as_str()) != Some(ItemType::MCP_LIST_TOOLS)
+        });
+
+        let list_tools_item = build_mcp_list_tools_item(mcp, server_label, server_keys);
+        output_array.insert(0, list_tools_item);
+
+        let mcp_call_items =
+            build_executed_mcp_call_items(&state.conversation_history, server_label);
+        let mut insert_pos = 1;
+        for item in mcp_call_items {
+            output_array.insert(insert_pos, item);
+            insert_pos += 1;
+        }
+    } else if let Some(obj) = response.as_object_mut() {
+        let mut output_items = Vec::new();
+        output_items.push(build_mcp_list_tools_item(mcp, server_label, server_keys));
+        output_items.extend(build_executed_mcp_call_items(
+            &state.conversation_history,
+            server_label,
+        ));
+        obj.insert("output".to_string(), Value::Array(output_items));
+    }
+}
+
+// ============================================================================
+// Tool Loop Execution
+// ============================================================================
+
+/// Execute the tool calling loop
+pub(super) async fn execute_tool_loop(
+    client: &reqwest::Client,
+    url: &str,
+    headers: Option<&HeaderMap>,
+    initial_payload: Value,
+    original_body: &ResponsesRequest,
+    active_mcp: &Arc<mcp::McpManager>,
+    config: &McpLoopConfig,
+) -> Result<Value, String> {
+    let mut state = ToolLoopState::new(original_body.input.clone());
+
+    // Get max_tool_calls from request (None means no user-specified limit)
+    let max_tool_calls = original_body.max_tool_calls.map(|n| n as usize);
+
+    // Keep initial_payload as base template (already has fields cleaned)
+    let base_payload = initial_payload.clone();
+    let tools_json = base_payload.get("tools").cloned().unwrap_or(json!([]));
+    let mut current_payload = initial_payload;
+
+    info!(
+        "Starting tool loop: max_tool_calls={:?}, max_iterations={}",
+        max_tool_calls, config.max_iterations
+    );
+
+    loop {
+        // Make request to upstream
+        let request_builder = client.post(url).json(&current_payload);
+        let request_builder = if let Some(headers) = headers {
+            apply_request_headers(headers, request_builder, true)
+        } else {
+            request_builder
+        };
+
+        let response = request_builder
+            .send()
+            .await
+            .map_err(|e| format!("upstream request failed: {}", e))?;
+
+        if !response.status().is_success() {
+            let status = response.status();
+            let body = response.text().await.unwrap_or_default();
+            return Err(format!("upstream error {}: {}", status, body));
+        }
+
+        let mut response_json = response
+            .json::<Value>()
+            .await
+            .map_err(|e| format!("parse response: {}", e))?;
+
+        // Check for function call
+        if let Some((call_id, tool_name, args_json_str)) = extract_function_call(&response_json) {
+            state.iteration += 1;
+            state.total_calls += 1;
+
+            info!(
+                "Tool loop iteration {}: calling {} (call_id: {})",
+                state.iteration, tool_name, call_id
+            );
+
+            // Check combined limit: use minimum of user's max_tool_calls (if set) and safety max_iterations
+            let effective_limit = match max_tool_calls {
+                Some(user_max) => user_max.min(config.max_iterations),
+                None => config.max_iterations,
+            };
+
+            if state.total_calls > effective_limit {
+                if let Some(user_max) = max_tool_calls {
+                    if state.total_calls > user_max {
+                        warn!("Reached user-specified max_tool_calls limit: {}", user_max);
+                    } else {
+                        warn!(
+                            "Reached safety max_iterations limit: {}",
+                            config.max_iterations
+                        );
+                    }
+                } else {
+                    warn!(
+                        "Reached safety max_iterations limit: {}",
+                        config.max_iterations
+                    );
+                }
+
+                return build_incomplete_response(
+                    response_json,
+                    state,
+                    "max_tool_calls",
+                    active_mcp,
+                    original_body,
+                    &config.server_keys,
+                );
+            }
+
+            // Execute tool - manager handles parsing and type coercion
+            debug!(
+                "Calling MCP tool '{}' with args: {}",
+                tool_name, args_json_str
+            );
+            let call_result = active_mcp
+                .call_tool(&tool_name, args_json_str.as_str())
+                .await;
+
+            let output_str = match call_result {
+                Ok(result) => match serde_json::to_string(&result) {
+                    Ok(output) => output,
+                    Err(e) => {
+                        warn!("Failed to serialize tool result: {}", e);
+                        json!({ "error": format!("Serialization error: {}", e) }).to_string()
+                    }
+                },
+                Err(err) => {
+                    warn!("Tool execution failed: {}", err);
+                    // Return error as output, let model decide how to proceed
+                    json!({ "error": format!("tool call failed: {}", err) }).to_string()
+                }
+            };
+
+            // Record the call
+            state.record_call(call_id, tool_name, args_json_str, output_str);
+
+            // Build resume payload
+            current_payload = build_resume_payload(
+                &base_payload,
+                &state.conversation_history,
+                &state.original_input,
+                &tools_json,
+                false, // is_streaming = false (non-streaming tool loop)
+            )?;
+        } else {
+            // No more tool calls, we're done
+            info!(
+                "Tool loop completed: {} iterations, {} total calls",
+                state.iteration, state.total_calls
+            );
+
+            // Inject MCP output items if we executed any tools
+            if state.total_calls > 0 {
+                let server_label = extract_server_label(original_body.tools.as_deref(), "mcp");
+
+                // Build mcp_list_tools item
+                let list_tools_item =
+                    build_mcp_list_tools_item(active_mcp, &server_label, &config.server_keys);
+
+                // Insert at beginning of output array
+                if let Some(output_array) = response_json
+                    .get_mut("output")
+                    .and_then(|v| v.as_array_mut())
+                {
+                    output_array.insert(0, list_tools_item);
+
+                    // Build mcp_call items using helper function
+                    let mcp_call_items =
+                        build_executed_mcp_call_items(&state.conversation_history, &server_label);
+
+                    // Insert mcp_call items after mcp_list_tools using mutable position
+                    let mut insert_pos = 1;
+                    for item in mcp_call_items {
+                        output_array.insert(insert_pos, item);
+                        insert_pos += 1;
+                    }
+                }
+            }
+
+            return Ok(response_json);
+        }
+    }
+}
+
+/// Build an incomplete response when limits are exceeded
+pub(super) fn build_incomplete_response(
+    mut response: Value,
+    state: ToolLoopState,
+    reason: &str,
+    active_mcp: &Arc<mcp::McpManager>,
+    original_body: &ResponsesRequest,
+    server_keys: &[String],
+) -> Result<Value, String> {
+    let obj = response
+        .as_object_mut()
+        .ok_or_else(|| "response not an object".to_string())?;
+
+    // Set status to completed (not failed - partial success)
+    obj.insert("status".to_string(), Value::String("completed".to_string()));
+
+    // Set incomplete_details
+    obj.insert(
+        "incomplete_details".to_string(),
+        json!({ "reason": reason }),
+    );
+
+    // Convert any function_call in output to mcp_call format
+    if let Some(output_array) = obj.get_mut("output").and_then(|v| v.as_array_mut()) {
+        let server_label = extract_server_label(original_body.tools.as_deref(), "mcp");
+
+        // Find any function_call items and convert them to mcp_call (incomplete)
+        let mut mcp_call_items = Vec::new();
+        for item in output_array.iter() {
+            let item_type = item.get("type").and_then(|t| t.as_str());
+            if item_type.is_some_and(is_function_call_type) {
+                let tool_name = item.get("name").and_then(|v| v.as_str()).unwrap_or("");
+                let args = item
+                    .get("arguments")
+                    .and_then(|v| v.as_str())
+                    .unwrap_or("{}");
+
+                // Mark as incomplete - not executed
+                let mcp_call_item = build_mcp_call_item(
+                    tool_name,
+                    args,
+                    "", // No output - wasn't executed
+                    &server_label,
+                    false, // Not successful
+                    Some("Not executed - response stopped due to limit"),
+                );
+                mcp_call_items.push(mcp_call_item);
+            }
+        }
+
+        // Add mcp_list_tools and executed mcp_call items at the beginning
+        if state.total_calls > 0 || !mcp_call_items.is_empty() {
+            let list_tools_item = build_mcp_list_tools_item(active_mcp, &server_label, server_keys);
+            output_array.insert(0, list_tools_item);
+
+            // Add mcp_call items for executed calls using helper
+            let executed_items =
+                build_executed_mcp_call_items(&state.conversation_history, &server_label);
+
+            let mut insert_pos = 1;
+            for item in executed_items {
+                output_array.insert(insert_pos, item);
+                insert_pos += 1;
+            }
+
+            // Add incomplete mcp_call items
+            for item in mcp_call_items {
+                output_array.insert(insert_pos, item);
+                insert_pos += 1;
+            }
+        }
+    }
+
+    // Add warning to metadata
+    if let Some(metadata_val) = obj.get_mut("metadata") {
+        if let Some(metadata_obj) = metadata_val.as_object_mut() {
+            if let Some(mcp_val) = metadata_obj.get_mut("mcp") {
+                if let Some(mcp_obj) = mcp_val.as_object_mut() {
+                    mcp_obj.insert(
+                        "truncation_warning".to_string(),
+                        Value::String(format!(
+                            "Loop terminated at {} iterations, {} total calls (reason: {})",
+                            state.iteration, state.total_calls, reason
+                        )),
+                    );
+                }
+            }
+        }
+    }
+
+    Ok(response)
+}
+
+// ============================================================================
+// Output Item Builders
+// ============================================================================
+
+/// Build a mcp_list_tools output item
+pub(super) fn build_mcp_list_tools_item(
+    mcp: &Arc<mcp::McpManager>,
+    server_label: &str,
+    server_keys: &[String],
+) -> Value {
+    let tools = mcp.list_tools_for_servers(server_keys);
+    let tools_json: Vec<Value> = tools
+        .iter()
+        .map(|t| {
+            json!({
+                "name": t.name,
+                "description": t.description,
+                "input_schema": Value::Object((*t.input_schema).clone()),
+                "annotations": {
+                    "read_only": false
+                }
+            })
+        })
+        .collect();
+
+    json!({
+        "id": generate_id("mcpl"),
+        "type": ItemType::MCP_LIST_TOOLS,
+        "server_label": server_label,
+        "tools": tools_json
+    })
+}
+
+/// Build a mcp_call output item
+pub(super) fn build_mcp_call_item(
+    tool_name: &str,
+    arguments: &str,
+    output: &str,
+    server_label: &str,
+    success: bool,
+    error: Option<&str>,
+) -> Value {
+    json!({
+        "id": generate_id("mcp"),
+        "type": ItemType::MCP_CALL,
+        "status": if success { "completed" } else { "failed" },
+        "approval_request_id": Value::Null,
+        "arguments": arguments,
+        "error": error,
+        "name": tool_name,
+        "output": output,
+        "server_label": server_label
+    })
+}
+
+/// Helper function to build mcp_call items from executed tool calls in conversation history
+pub(super) fn build_executed_mcp_call_items(
+    conversation_history: &[Value],
+    server_label: &str,
+) -> Vec<Value> {
+    let mut mcp_call_items = Vec::new();
+
+    for item in conversation_history {
+        if item.get("type").and_then(|t| t.as_str()) == Some(ItemType::FUNCTION_CALL) {
+            let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or("");
+            let tool_name = item.get("name").and_then(|v| v.as_str()).unwrap_or("");
+            let args = item
+                .get("arguments")
+                .and_then(|v| v.as_str())
+                .unwrap_or("{}");
+
+            // Find corresponding output
+            let output_item = conversation_history.iter().find(|o| {
+                o.get("type").and_then(|t| t.as_str()) == Some("function_call_output")
+                    && o.get("call_id").and_then(|c| c.as_str()) == Some(call_id)
+            });
+
+            let output_str = output_item
+                .and_then(|o| o.get("output").and_then(|v| v.as_str()))
+                .unwrap_or("{}");
+
+            // Check if output contains error by parsing JSON
+            let is_error = serde_json::from_str::<Value>(output_str)
+                .map(|v| v.get("error").is_some())
+                .unwrap_or(false);
+
+            let mcp_call_item = build_mcp_call_item(
+                tool_name,
+                args,
+                output_str,
+                server_label,
+                !is_error,
+                if is_error {
+                    Some("Tool execution failed")
+                } else {
+                    None
+                },
+            );
+            mcp_call_items.push(mcp_call_item);
+        }
+    }
+
+    mcp_call_items
+}
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+/// Extract function call from a response
+pub(super) fn extract_function_call(resp: &Value) -> Option<(String, String, String)> {
+    let output = resp.get("output")?.as_array()?;
+    for item in output {
+        let obj = item.as_object()?;
+        let t = obj.get("type")?.as_str()?;
+        if is_function_call_type(t) {
+            let call_id = obj
+                .get("call_id")
+                .and_then(|v| v.as_str())
+                .map(|s| s.to_string())
+                .or_else(|| {
+                    obj.get("id")
+                        .and_then(|v| v.as_str())
+                        .map(|s| s.to_string())
+                })?;
+            let name = obj.get("name")?.as_str()?.to_string();
+            let arguments = obj.get("arguments")?.as_str()?.to_string();
+            return Some((call_id, name, arguments));
+        }
+    }
+    None
+}

third_party/sglang/sgl-model-gateway/src/routers/openai/responses/mod.rs

@@ -0,0 +1,19 @@
+//! OpenAI-compatible responses handling module
+//!
+//! This module provides comprehensive support for OpenAI Responses API with:
+//! - Streaming and non-streaming response handling
+//! - MCP (Model Context Protocol) tool interception and execution
+//! - SSE (Server-Sent Events) parsing and forwarding
+//! - Response accumulation for persistence
+//! - Tool call detection and output index remapping
+
+mod accumulator;
+mod common;
+mod mcp;
+mod non_streaming;
+mod streaming;
+mod tool_handler;
+mod utils;
+
+pub use non_streaming::handle_non_streaming_response;
+pub use streaming::handle_streaming_response;

third_party/sglang/sgl-model-gateway/src/routers/openai/responses/non_streaming.rs

@@ -0,0 +1,171 @@
+//! Non-streaming response handling for OpenAI-compatible responses
+//!
+//! This module handles non-streaming Responses API requests with MCP tool support.
+
+use axum::{
+    http::StatusCode,
+    response::{IntoResponse, Response},
+    Json,
+};
+use serde_json::{json, Value};
+use tracing::warn;
+
+use super::{
+    mcp::{execute_tool_loop, prepare_mcp_tools_as_functions},
+    utils::{mask_tools_as_mcp, patch_response_with_request_metadata},
+};
+use crate::routers::{
+    header_utils::{apply_provider_headers, extract_auth_header},
+    mcp_utils::{ensure_request_mcp_client, McpLoopConfig},
+    openai::context::{PayloadState, RequestContext},
+    persistence_utils::persist_conversation_items,
+};
+
+/// Handle a non-streaming responses request
+pub async fn handle_non_streaming_response(mut ctx: RequestContext) -> Response {
+    let payload_state = match ctx.state.payload.take() {
+        Some(ps) => ps,
+        None => {
+            return (StatusCode::INTERNAL_SERVER_ERROR, "Payload not prepared").into_response();
+        }
+    };
+
+    let PayloadState {
+        json: mut payload,
+        url,
+        previous_response_id,
+    } = payload_state;
+
+    let original_body = ctx.responses_request();
+    let worker = match ctx.worker() {
+        Some(w) => w.clone(),
+        None => {
+            return (StatusCode::INTERNAL_SERVER_ERROR, "Worker not selected").into_response();
+        }
+    };
+    let mcp_manager = match ctx.components.mcp_manager() {
+        Some(m) => m,
+        None => {
+            return (StatusCode::INTERNAL_SERVER_ERROR, "MCP manager required").into_response();
+        }
+    };
+
+    let server_keys = match original_body.tools.as_ref() {
+        Some(tools) => match ensure_request_mcp_client(mcp_manager, tools.as_slice()).await {
+            Some((_manager, keys)) => keys,
+            None => Vec::new(),
+        },
+        None => Vec::new(),
+    };
+
+    let active_mcp = if mcp_manager.list_tools_for_servers(&server_keys).is_empty() {
+        None
+    } else {
+        Some(mcp_manager)
+    };
+
+    let mut response_json: Value;
+
+    if let Some(mcp) = active_mcp {
+        let config = McpLoopConfig {
+            server_keys: server_keys.clone(),
+            ..McpLoopConfig::default()
+        };
+        prepare_mcp_tools_as_functions(&mut payload, mcp, &server_keys);
+
+        match execute_tool_loop(
+            ctx.components.client(),
+            &url,
+            ctx.headers(),
+            payload,
+            original_body,
+            mcp,
+            &config,
+        )
+        .await
+        {
+            Ok(resp) => response_json = resp,
+            Err(err) => {
+                worker.circuit_breaker().record_failure();
+                return (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    Json(json!({"error": {"message": err}})),
+                )
+                    .into_response();
+            }
+        }
+    } else {
+        let mut request_builder = ctx.components.client().post(&url).json(&payload);
+        let auth_header = extract_auth_header(ctx.headers(), worker.api_key());
+        request_builder = apply_provider_headers(request_builder, &url, auth_header.as_ref());
+
+        let response = match request_builder.send().await {
+            Ok(r) => r,
+            Err(e) => {
+                worker.circuit_breaker().record_failure();
+                tracing::error!(
+                    url = %url,
+                    error = %e,
+                    "Failed to forward request to OpenAI"
+                );
+                return (
+                    StatusCode::BAD_GATEWAY,
+                    format!("Failed to forward request to OpenAI: {}", e),
+                )
+                    .into_response();
+            }
+        };
+
+        if !response.status().is_success() {
+            worker.circuit_breaker().record_failure();
+            let status = StatusCode::from_u16(response.status().as_u16())
+                .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
+            let body = response.text().await.unwrap_or_default();
+            return (status, body).into_response();
+        }
+
+        response_json = match response.json::<Value>().await {
+            Ok(r) => r,
+            Err(e) => {
+                worker.circuit_breaker().record_failure();
+                return (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    format!("Failed to parse upstream response: {}", e),
+                )
+                    .into_response();
+            }
+        };
+
+        worker.circuit_breaker().record_success();
+    }
+
+    mask_tools_as_mcp(&mut response_json, original_body);
+    patch_response_with_request_metadata(
+        &mut response_json,
+        original_body,
+        previous_response_id.as_deref(),
+    );
+
+    if let Err(err) = persist_conversation_items(
+        ctx.components
+            .conversation_storage()
+            .expect("Conversation storage required")
+            .clone(),
+        ctx.components
+            .conversation_item_storage()
+            .expect("Conversation item storage required")
+            .clone(),
+        ctx.components
+            .response_storage()
+            .expect("Response storage required")
+            .clone(),
+        &response_json,
+        original_body,
+    )
+    .await
+    {
+        warn!("Failed to persist conversation items: {}", err);
+    }
+
+    (StatusCode::OK, Json(response_json)).into_response()
+}

third_party/sglang/sgl-model-gateway/src/routers/openai/responses/tool_handler.rs

@@ -0,0 +1,341 @@
+//! Streaming tool call handling for MCP interception.
+
+use std::collections::HashMap;
+
+use serde_json::Value;
+use tracing::warn;
+
+use super::{
+    accumulator::StreamingResponseAccumulator,
+    common::{extract_output_index, get_event_type},
+    mcp::FunctionCallInProgress,
+};
+use crate::protocols::event_types::{
+    is_function_call_type, FunctionCallEvent, OutputItemEvent, ResponseEvent,
+};
+
+// ============================================================================
+// Stream Action Enum
+// ============================================================================
+
+/// Action to take based on streaming event processing
+#[derive(Debug)]
+pub(super) enum StreamAction {
+    Forward,      // Pass event to client
+    Buffer,       // Accumulate for tool execution
+    ExecuteTools, // Function call complete, execute now
+}
+
+// ============================================================================
+// Output Index Mapper
+// ============================================================================
+
+/// Maps upstream output indices to sequential downstream indices
+#[derive(Debug, Default)]
+pub(super) struct OutputIndexMapper {
+    next_index: usize,
+    // Map upstream output_index -> remapped output_index
+    assigned: HashMap<usize, usize>,
+}
+
+impl OutputIndexMapper {
+    pub fn with_start(next_index: usize) -> Self {
+        Self {
+            next_index,
+            assigned: HashMap::new(),
+        }
+    }
+
+    pub fn ensure_mapping(&mut self, upstream_index: usize) -> usize {
+        *self.assigned.entry(upstream_index).or_insert_with(|| {
+            let assigned = self.next_index;
+            self.next_index += 1;
+            assigned
+        })
+    }
+
+    pub fn lookup(&self, upstream_index: usize) -> Option<usize> {
+        self.assigned.get(&upstream_index).copied()
+    }
+
+    pub fn allocate_synthetic(&mut self) -> usize {
+        let assigned = self.next_index;
+        self.next_index += 1;
+        assigned
+    }
+
+    pub fn next_index(&self) -> usize {
+        self.next_index
+    }
+}
+
+// ============================================================================
+// Streaming Tool Handler
+// ============================================================================
+
+/// Handles streaming responses with MCP tool call interception
+pub(super) struct StreamingToolHandler {
+    /// Accumulator for response persistence
+    pub accumulator: StreamingResponseAccumulator,
+    /// Function calls being built from deltas
+    pub pending_calls: Vec<FunctionCallInProgress>,
+    /// Track if we're currently in a function call
+    in_function_call: bool,
+    /// Manage output_index remapping so they increment per item
+    output_index_mapper: OutputIndexMapper,
+    /// Original response id captured from the first response.created event
+    pub original_response_id: Option<String>,
+}
+
+impl StreamingToolHandler {
+    pub fn with_starting_index(start: usize) -> Self {
+        Self {
+            accumulator: StreamingResponseAccumulator::new(),
+            pending_calls: Vec::new(),
+            in_function_call: false,
+            output_index_mapper: OutputIndexMapper::with_start(start),
+            original_response_id: None,
+        }
+    }
+
+    pub fn ensure_output_index(&mut self, upstream_index: usize) -> usize {
+        self.output_index_mapper.ensure_mapping(upstream_index)
+    }
+
+    pub fn mapped_output_index(&self, upstream_index: usize) -> Option<usize> {
+        self.output_index_mapper.lookup(upstream_index)
+    }
+
+    pub fn allocate_synthetic_output_index(&mut self) -> usize {
+        self.output_index_mapper.allocate_synthetic()
+    }
+
+    pub fn next_output_index(&self) -> usize {
+        self.output_index_mapper.next_index()
+    }
+
+    pub fn original_response_id(&self) -> Option<&str> {
+        self.original_response_id
+            .as_deref()
+            .or_else(|| self.accumulator.original_response_id())
+    }
+
+    pub fn snapshot_final_response(&self) -> Option<Value> {
+        self.accumulator.snapshot_final_response()
+    }
+
+    /// Process an SSE event and determine what action to take
+    pub fn process_event(&mut self, event_name: Option<&str>, data: &str) -> StreamAction {
+        // Always feed to accumulator for storage
+        self.accumulator.ingest_block(&format!(
+            "{}data: {}",
+            event_name
+                .map(|n| format!("event: {}\n", n))
+                .unwrap_or_default(),
+            data
+        ));
+
+        let parsed: Value = match serde_json::from_str(data) {
+            Ok(v) => v,
+            Err(_) => return StreamAction::Forward,
+        };
+
+        match get_event_type(event_name, &parsed) {
+            ResponseEvent::CREATED => {
+                if self.original_response_id.is_none() {
+                    self.original_response_id = parsed
+                        .get("response")
+                        .and_then(|v| v.get("id"))
+                        .and_then(|v| v.as_str())
+                        .map(|s| s.to_string());
+                }
+                StreamAction::Forward
+            }
+            ResponseEvent::COMPLETED => StreamAction::Forward,
+            OutputItemEvent::ADDED => self.handle_output_item_added(&parsed),
+            FunctionCallEvent::ARGUMENTS_DELTA => self.handle_arguments_delta(&parsed),
+            FunctionCallEvent::ARGUMENTS_DONE => self.handle_arguments_done(&parsed),
+            OutputItemEvent::DELTA => self.process_output_delta(&parsed),
+            OutputItemEvent::DONE => {
+                if let Some(output_index) = extract_output_index(&parsed) {
+                    self.ensure_output_index(output_index);
+                }
+                if self.has_complete_calls() {
+                    StreamAction::ExecuteTools
+                } else {
+                    StreamAction::Forward
+                }
+            }
+            _ => StreamAction::Forward,
+        }
+    }
+
+    fn handle_output_item_added(&mut self, parsed: &Value) -> StreamAction {
+        if let Some(output_index) = extract_output_index(parsed) {
+            self.ensure_output_index(output_index);
+        }
+
+        // Check if this is a function_call item being added
+        let Some(item) = parsed.get("item") else {
+            return StreamAction::Forward;
+        };
+        let Some(item_type) = item.get("type").and_then(|v| v.as_str()) else {
+            return StreamAction::Forward;
+        };
+
+        if !is_function_call_type(item_type) {
+            return StreamAction::Forward;
+        }
+
+        let Some(output_index) = extract_output_index(parsed) else {
+            warn!(
+                "Missing output_index in function_call added event, \
+                 forwarding without processing for tool execution"
+            );
+            return StreamAction::Forward;
+        };
+
+        let assigned_index = self.ensure_output_index(output_index);
+        let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or("");
+        let name = item.get("name").and_then(|v| v.as_str()).unwrap_or("");
+
+        let call = self.get_or_create_call(output_index, item);
+        call.call_id = call_id.to_string();
+        call.name = name.to_string();
+        call.assigned_output_index = Some(assigned_index);
+        self.in_function_call = true;
+
+        StreamAction::Forward
+    }
+
+    fn handle_arguments_delta(&mut self, parsed: &Value) -> StreamAction {
+        let Some(output_index) = extract_output_index(parsed) else {
+            return StreamAction::Forward;
+        };
+
+        let assigned_index = self.ensure_output_index(output_index);
+
+        if let Some(delta) = parsed.get("delta").and_then(|v| v.as_str()) {
+            if let Some(call) = self.find_call_mut(output_index) {
+                call.arguments_buffer.push_str(delta);
+                if let Some(obfuscation) = parsed.get("obfuscation").and_then(|v| v.as_str()) {
+                    call.last_obfuscation = Some(obfuscation.to_string());
+                }
+                if call.assigned_output_index.is_none() {
+                    call.assigned_output_index = Some(assigned_index);
+                }
+            }
+        }
+        StreamAction::Forward
+    }
+
+    fn handle_arguments_done(&mut self, parsed: &Value) -> StreamAction {
+        if let Some(output_index) = extract_output_index(parsed) {
+            let assigned_index = self.ensure_output_index(output_index);
+            if let Some(call) = self.find_call_mut(output_index) {
+                if call.assigned_output_index.is_none() {
+                    call.assigned_output_index = Some(assigned_index);
+                }
+            }
+        }
+
+        if self.has_complete_calls() {
+            StreamAction::ExecuteTools
+        } else {
+            StreamAction::Forward
+        }
+    }
+
+    fn find_call_mut(&mut self, output_index: usize) -> Option<&mut FunctionCallInProgress> {
+        self.pending_calls
+            .iter_mut()
+            .find(|c| c.output_index == output_index)
+    }
+
+    /// Process output delta events to detect and accumulate function calls
+    fn process_output_delta(&mut self, event: &Value) -> StreamAction {
+        let output_index = extract_output_index(event).unwrap_or(0);
+        let assigned_index = self.ensure_output_index(output_index);
+
+        let delta = match event.get("delta") {
+            Some(d) => d,
+            None => return StreamAction::Forward,
+        };
+
+        // Check if this is a function call delta
+        let item_type = delta.get("type").and_then(|v| v.as_str());
+
+        if item_type.is_some_and(is_function_call_type) {
+            self.in_function_call = true;
+
+            // Get or create function call for this output index
+            let call = self.get_or_create_call(output_index, delta);
+            call.assigned_output_index = Some(assigned_index);
+
+            // Accumulate call_id if present
+            if let Some(call_id) = delta.get("call_id").and_then(|v| v.as_str()) {
+                call.call_id = call_id.to_string();
+            }
+
+            // Accumulate name if present
+            if let Some(name) = delta.get("name").and_then(|v| v.as_str()) {
+                call.name.push_str(name);
+            }
+
+            // Accumulate arguments if present
+            if let Some(args) = delta.get("arguments").and_then(|v| v.as_str()) {
+                call.arguments_buffer.push_str(args);
+            }
+
+            if let Some(obfuscation) = delta.get("obfuscation").and_then(|v| v.as_str()) {
+                call.last_obfuscation = Some(obfuscation.to_string());
+            }
+
+            // Buffer this event, don't forward to client
+            return StreamAction::Buffer;
+        }
+
+        // Forward non-function-call events
+        StreamAction::Forward
+    }
+
+    fn get_or_create_call(
+        &mut self,
+        output_index: usize,
+        delta: &Value,
+    ) -> &mut FunctionCallInProgress {
+        // Find existing call for this output index
+        if let Some(pos) = self
+            .pending_calls
+            .iter()
+            .position(|c| c.output_index == output_index)
+        {
+            return &mut self.pending_calls[pos];
+        }
+
+        // Create new call
+        let call_id = delta
+            .get("call_id")
+            .and_then(|v| v.as_str())
+            .unwrap_or("")
+            .to_string();
+
+        let mut call = FunctionCallInProgress::new(call_id, output_index);
+        if let Some(obfuscation) = delta.get("obfuscation").and_then(|v| v.as_str()) {
+            call.last_obfuscation = Some(obfuscation.to_string());
+        }
+
+        self.pending_calls.push(call);
+        self.pending_calls
+            .last_mut()
+            .expect("Just pushed to pending_calls, must have at least one element")
+    }
+
+    fn has_complete_calls(&self) -> bool {
+        !self.pending_calls.is_empty() && self.pending_calls.iter().all(|c| c.is_complete())
+    }
+
+    pub fn take_pending_calls(&mut self) -> Vec<FunctionCallInProgress> {
+        std::mem::take(&mut self.pending_calls)
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/openai/responses/utils.rs

@@ -0,0 +1,235 @@
+//! Response patching and transformation utilities for OpenAI responses
+
+use serde_json::{json, Map, Value};
+use tracing::warn;
+
+use crate::protocols::{
+    event_types::is_response_event,
+    responses::{ResponseToolType, ResponsesRequest},
+};
+
+/// Check if a JSON value is missing, null, or an empty string
+fn is_missing_or_empty(value: Option<&Value>) -> bool {
+    match value {
+        None => true,
+        Some(v) => v.is_null() || v.as_str().is_some_and(|s| s.is_empty()),
+    }
+}
+
+/// Insert a string value into a JSON object if the condition is met
+fn insert_if<F>(obj: &mut Map<String, Value>, key: &str, value: &str, condition: F)
+where
+    F: FnOnce(&Map<String, Value>) -> bool,
+{
+    if condition(obj) {
+        obj.insert(key.to_string(), Value::String(value.to_string()));
+    }
+}
+
+/// Patch response JSON with metadata from original request
+///
+/// The upstream response may be missing fields that were in the original request.
+/// This function ensures these fields are preserved in the final response:
+/// - `previous_response_id` - conversation threading
+/// - `instructions` - system instructions
+/// - `metadata` - user-provided metadata
+/// - `store` - whether to persist the response
+/// - `model` - model identifier
+/// - `safety_identifier` - user identifier for safety
+pub(super) fn patch_response_with_request_metadata(
+    response_json: &mut Value,
+    original_body: &ResponsesRequest,
+    original_previous_response_id: Option<&str>,
+) {
+    let Some(obj) = response_json.as_object_mut() else {
+        return;
+    };
+
+    // Set previous_response_id if missing/empty
+    if let Some(prev_id) = original_previous_response_id {
+        insert_if(obj, "previous_response_id", prev_id, |o| {
+            is_missing_or_empty(o.get("previous_response_id"))
+        });
+    }
+
+    // Set instructions if missing/null
+    if let Some(instructions) = &original_body.instructions {
+        insert_if(obj, "instructions", instructions, |o| {
+            is_missing_or_empty(o.get("instructions"))
+        });
+    }
+
+    // Set metadata if missing/null
+    if is_missing_or_empty(obj.get("metadata")) {
+        if let Some(metadata) = &original_body.metadata {
+            let metadata_map: Map<String, Value> = metadata
+                .iter()
+                .map(|(k, v)| (k.clone(), v.clone()))
+                .collect();
+            obj.insert("metadata".to_string(), Value::Object(metadata_map));
+        }
+    }
+
+    // Always set store
+    obj.insert(
+        "store".to_string(),
+        Value::Bool(original_body.store.unwrap_or(false)),
+    );
+
+    // Set model if missing/empty
+    insert_if(obj, "model", &original_body.model, |o| {
+        is_missing_or_empty(o.get("model"))
+    });
+
+    // Set safety_identifier if null (but key exists)
+    if let Some(user) = &original_body.user {
+        if obj
+            .get("safety_identifier")
+            .is_some_and(|v: &Value| v.is_null())
+        {
+            obj.insert("safety_identifier".to_string(), Value::String(user.clone()));
+        }
+    }
+
+    // Attach conversation id for client response
+    if let Some(conv_id) = &original_body.conversation {
+        obj.insert("conversation".to_string(), json!({ "id": conv_id }));
+    }
+}
+
+/// Extract data payload from SSE block lines
+fn extract_sse_data(block: &str) -> Option<String> {
+    let data_lines: Vec<_> = block
+        .lines()
+        .filter(|line| line.starts_with("data:"))
+        .map(|line| line.trim_start_matches("data:").trim_start())
+        .collect();
+
+    if data_lines.is_empty() {
+        None
+    } else {
+        Some(data_lines.join("\n"))
+    }
+}
+
+/// Rebuild SSE block with new data payload
+fn rebuild_sse_block(block: &str, new_payload: &str) -> String {
+    let mut rebuilt_lines = Vec::new();
+    let mut data_written = false;
+
+    for line in block.lines() {
+        if line.starts_with("data:") {
+            if !data_written {
+                rebuilt_lines.push(format!("data: {}", new_payload));
+                data_written = true;
+            }
+        } else {
+            rebuilt_lines.push(line.to_string());
+        }
+    }
+
+    if !data_written {
+        rebuilt_lines.push(format!("data: {}", new_payload));
+    }
+
+    rebuilt_lines.join("\n")
+}
+
+/// Rewrite streaming SSE block to include metadata from original request
+pub(super) fn rewrite_streaming_block(
+    block: &str,
+    original_body: &ResponsesRequest,
+    original_previous_response_id: Option<&str>,
+) -> Option<String> {
+    let trimmed = block.trim();
+    if trimmed.is_empty() {
+        return None;
+    }
+
+    let payload = extract_sse_data(trimmed)?;
+    let mut parsed: Value = serde_json::from_str(&payload)
+        .map_err(|e| warn!("Failed to parse streaming JSON payload: {}", e))
+        .ok()?;
+
+    let event_type = parsed
+        .get("type")
+        .and_then(|v| v.as_str())
+        .unwrap_or_default();
+
+    if !is_response_event(event_type) {
+        return None;
+    }
+
+    let response_obj = parsed.get_mut("response").and_then(|v| v.as_object_mut())?;
+    let mut changed = false;
+
+    // Update store value if different
+    let desired_store = Value::Bool(original_body.store.unwrap_or(false));
+    if response_obj.get("store") != Some(&desired_store) {
+        response_obj.insert("store".to_string(), desired_store);
+        changed = true;
+    }
+
+    // Set previous_response_id if missing/empty
+    if let Some(prev_id) = original_previous_response_id {
+        if is_missing_or_empty(response_obj.get("previous_response_id")) {
+            response_obj.insert("previous_response_id".to_string(), json!(prev_id));
+            changed = true;
+        }
+    }
+
+    // Attach conversation id
+    if let Some(conv_id) = &original_body.conversation {
+        response_obj.insert("conversation".to_string(), json!({ "id": conv_id }));
+        changed = true;
+    }
+
+    if !changed {
+        return None;
+    }
+
+    let new_payload = serde_json::to_string(&parsed)
+        .map_err(|e| warn!("Failed to serialize modified streaming payload: {}", e))
+        .ok()?;
+
+    Some(rebuild_sse_block(trimmed, &new_payload))
+}
+
+/// Helper to insert an optional string field into a JSON map
+fn insert_optional_string(map: &mut Map<String, Value>, key: &str, value: &Option<String>) {
+    if let Some(v) = value {
+        map.insert(key.to_string(), Value::String(v.clone()));
+    }
+}
+
+/// Mask function tools as MCP tools in response for client
+pub(super) fn mask_tools_as_mcp(resp: &mut Value, original_body: &ResponsesRequest) {
+    let mcp_tool = original_body.tools.as_ref().and_then(|tools| {
+        tools
+            .iter()
+            .find(|t| matches!(t.r#type, ResponseToolType::Mcp) && t.server_url.is_some())
+    });
+
+    let Some(t) = mcp_tool else {
+        return;
+    };
+
+    let mut m = Map::new();
+    m.insert("type".to_string(), json!("mcp"));
+    insert_optional_string(&mut m, "server_label", &t.server_label);
+    insert_optional_string(&mut m, "server_url", &t.server_url);
+    insert_optional_string(&mut m, "server_description", &t.server_description);
+    insert_optional_string(&mut m, "require_approval", &t.require_approval);
+
+    if let Some(allowed) = &t.allowed_tools {
+        m.insert(
+            "allowed_tools".to_string(),
+            Value::Array(allowed.iter().map(|s| json!(s)).collect()),
+        );
+    }
+
+    if let Some(obj) = resp.as_object_mut() {
+        obj.insert("tools".to_string(), json!([Value::Object(m)]));
+        obj.entry("tool_choice").or_insert(json!("auto"));
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/parse/handlers.rs

@@ -0,0 +1,104 @@
+//! Parser handlers for function calls and reasoning extraction
+
+use std::sync::Arc;
+
+use axum::{
+    http::StatusCode,
+    response::{IntoResponse, Response},
+    Json,
+};
+use tracing::error;
+
+use crate::{
+    app_context::AppContext,
+    protocols::parser::{ParseFunctionCallRequest, SeparateReasoningRequest},
+};
+
+/// Helper to create error responses
+fn error_response(status: StatusCode, message: &str) -> Response {
+    (
+        status,
+        Json(serde_json::json!({
+            "error": message,
+            "success": false
+        })),
+    )
+        .into_response()
+}
+
+/// Parse function calls from model output text
+pub async fn parse_function_call(
+    ctx: &Arc<AppContext>,
+    req: &ParseFunctionCallRequest,
+) -> Response {
+    let Some(factory) = &ctx.tool_parser_factory else {
+        return error_response(
+            StatusCode::SERVICE_UNAVAILABLE,
+            "Tool parser factory not initialized",
+        );
+    };
+
+    let Some(pooled_parser) = factory.registry().get_pooled_parser(&req.tool_call_parser) else {
+        return error_response(
+            StatusCode::BAD_REQUEST,
+            &format!("Unknown tool parser: {}", req.tool_call_parser),
+        );
+    };
+
+    let parser = pooled_parser.lock().await;
+    match parser.parse_complete(&req.text).await {
+        Ok((remaining_text, tool_calls)) => (
+            StatusCode::OK,
+            Json(serde_json::json!({
+                "remaining_text": remaining_text,
+                "tool_calls": tool_calls,
+                "success": true
+            })),
+        )
+            .into_response(),
+        Err(e) => {
+            error!("Failed to parse function calls: {}", e);
+            error_response(
+                StatusCode::BAD_REQUEST,
+                &format!("Failed to parse function calls: {}", e),
+            )
+        }
+    }
+}
+
+/// Parse and separate reasoning from normal text
+pub async fn parse_reasoning(ctx: &Arc<AppContext>, req: &SeparateReasoningRequest) -> Response {
+    let Some(factory) = &ctx.reasoning_parser_factory else {
+        return error_response(
+            StatusCode::SERVICE_UNAVAILABLE,
+            "Reasoning parser factory not initialized",
+        );
+    };
+
+    let Some(pooled_parser) = factory.registry().get_pooled_parser(&req.reasoning_parser) else {
+        return error_response(
+            StatusCode::BAD_REQUEST,
+            &format!("Unknown reasoning parser: {}", req.reasoning_parser),
+        );
+    };
+
+    let mut parser = pooled_parser.lock().await;
+    match parser.detect_and_parse_reasoning(&req.text) {
+        Ok(result) => (
+            StatusCode::OK,
+            Json(serde_json::json!({
+                "normal_text": result.normal_text,
+                "reasoning_text": result.reasoning_text,
+                "success": true
+            })),
+        )
+            .into_response(),
+        Err(e) => {
+            error!("Failed to separate reasoning: {}", e);
+            error_response(
+                StatusCode::BAD_REQUEST,
+                &format!("Failed to separate reasoning: {}", e),
+            )
+        }
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/parse/mod.rs

@@ -0,0 +1,9 @@
+//! Parser module for function calls and reasoning extraction
+//!
+//! This module provides parsing operations for model output, including:
+//! - Function call extraction from text
+//! - Reasoning separation from normal text
+
+mod handlers;
+
+pub use handlers::{parse_function_call, parse_reasoning};

third_party/sglang/sgl-model-gateway/src/routers/persistence_utils.rs

@@ -0,0 +1,404 @@
+//! Utilities for persisting responses and conversation items across router implementations.
+
+use std::sync::Arc;
+
+use chrono::Utc;
+use data_connector::{
+    ConversationId, ConversationItem, ConversationItemId, ConversationItemStorage,
+    ConversationStorage, NewConversationItem, ResponseId, ResponseStorage, StoredResponse,
+};
+use serde_json::{json, Value};
+use tracing::{debug, info, warn};
+
+use crate::protocols::responses::{
+    generate_id, ResponseInput, ResponseInputOutputItem, ResponsesRequest, StringOrContentParts,
+};
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/// Field mappings for item types that store data in content
+pub const ITEM_TYPE_FIELDS: &[(&str, &[&str])] = &[
+    (
+        "mcp_call",
+        &[
+            "name",
+            "arguments",
+            "output",
+            "server_label",
+            "approval_request_id",
+            "error",
+        ],
+    ),
+    ("mcp_list_tools", &["tools", "server_label"]),
+    ("function_call", &["call_id", "name", "arguments", "output"]),
+    ("function_call_output", &["call_id", "output"]),
+];
+
+// ============================================================================
+// JSON Serialization
+// ============================================================================
+
+/// Convert a ConversationItem to JSON, extracting specified fields based on item type
+/// or including content as-is for standard message types.
+pub fn item_to_json(item: &ConversationItem) -> Value {
+    let mut obj = serde_json::Map::new();
+    obj.insert("id".to_string(), json!(item.id.0));
+    obj.insert("type".to_string(), json!(item.item_type));
+
+    if let Some(role) = &item.role {
+        obj.insert("role".to_string(), json!(role));
+    }
+
+    // Find field mappings for this item type
+    let fields = ITEM_TYPE_FIELDS
+        .iter()
+        .find(|(t, _)| *t == item.item_type)
+        .map(|(_, fields)| *fields);
+
+    if let Some(fields) = fields {
+        // Extract specific fields from content
+        if let Some(content_obj) = item.content.as_object() {
+            for field in fields {
+                if let Some(value) = content_obj.get(*field) {
+                    obj.insert((*field).to_string(), value.clone());
+                }
+            }
+        }
+    } else {
+        // Default: include content as-is
+        obj.insert("content".to_string(), item.content.clone());
+    }
+
+    if let Some(status) = &item.status {
+        obj.insert("status".to_string(), json!(status));
+    }
+
+    Value::Object(obj)
+}
+
+// ============================================================================
+// Item Creation Helper
+// ============================================================================
+
+/// Create a conversation item and optionally link it to a conversation.
+/// Sets default "completed" status if not provided.
+pub async fn create_and_link_item(
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conv_id_opt: Option<&ConversationId>,
+    mut new_item: NewConversationItem,
+) -> Result<(), String> {
+    if new_item.status.is_none() {
+        new_item.status = Some("completed".to_string());
+    }
+
+    let created = item_storage
+        .create_item(new_item)
+        .await
+        .map_err(|e| format!("Failed to create item: {e}"))?;
+
+    if let Some(conv_id) = conv_id_opt {
+        item_storage
+            .link_item(conv_id, &created.id, Utc::now())
+            .await
+            .map_err(|e| format!("Failed to link item: {e}"))?;
+
+        debug!(
+            conversation_id = %conv_id.0,
+            item_id = %created.id.0,
+            item_type = %created.item_type,
+            "Persisted conversation item and link"
+        );
+    } else {
+        debug!(
+            item_id = %created.id.0,
+            item_type = %created.item_type,
+            "Persisted conversation item (no conversation link)"
+        );
+    }
+
+    Ok(())
+}
+
+// ============================================================================
+// Response Persistence
+// ============================================================================
+
+/// Extract a string field from JSON, returning owned String
+fn get_string(json: &Value, key: &str) -> Option<String> {
+    json.get(key).and_then(|v| v.as_str()).map(String::from)
+}
+
+/// Build a StoredResponse from response JSON and original request
+pub fn build_stored_response(
+    response_json: &Value,
+    original_body: &ResponsesRequest,
+) -> StoredResponse {
+    let mut stored = StoredResponse::new(None);
+
+    // Initialize empty arrays - will be populated by persist_conversation_items
+    stored.input = Value::Array(vec![]);
+    stored.output = Value::Array(vec![]);
+
+    stored.instructions =
+        get_string(response_json, "instructions").or_else(|| original_body.instructions.clone());
+
+    stored.model = get_string(response_json, "model").or_else(|| Some(original_body.model.clone()));
+
+    stored.safety_identifier = original_body.user.clone();
+    stored.conversation_id = original_body.conversation.clone();
+
+    stored.metadata = response_json
+        .get("metadata")
+        .and_then(|v| v.as_object())
+        .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
+        .unwrap_or_else(|| original_body.metadata.clone().unwrap_or_default());
+
+    stored.previous_response_id = get_string(response_json, "previous_response_id")
+        .map(|s| ResponseId::from(s.as_str()))
+        .or_else(|| {
+            original_body
+                .previous_response_id
+                .as_deref()
+                .map(ResponseId::from)
+        });
+
+    if let Some(id_str) = get_string(response_json, "id") {
+        stored.id = ResponseId::from(id_str.as_str());
+    }
+
+    stored.raw_response = response_json.clone();
+    stored
+}
+
+/// Extract and normalize input items from ResponseInput
+fn extract_input_items(input: &ResponseInput) -> Result<Vec<Value>, String> {
+    let items = match input {
+        ResponseInput::Text(text) => {
+            // Convert simple text to message item
+            vec![json!({
+                "id": generate_id("msg"),
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": text}],
+                "status": "completed"
+            })]
+        }
+        ResponseInput::Items(items) => {
+            // Process all item types and ensure IDs
+            items
+                .iter()
+                .map(|item| {
+                    match item {
+                        ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => {
+                            // Convert SimpleInputMessage to standard message format with ID
+                            let content_json = match content {
+                                StringOrContentParts::String(s) => {
+                                    json!([{"type": "input_text", "text": s}])
+                                }
+                                StringOrContentParts::Array(parts) => serde_json::to_value(parts)
+                                    .map_err(|e| {
+                                    format!("Failed to serialize content: {}", e)
+                                })?,
+                            };
+
+                            Ok(json!({
+                                "id": generate_id("msg"),
+                                "type": "message",
+                                "role": role,
+                                "content": content_json,
+                                "status": "completed"
+                            }))
+                        }
+                        _ => {
+                            // For other item types, serialize and ensure ID
+                            let mut value = serde_json::to_value(item)
+                                .map_err(|e| format!("Failed to serialize item: {}", e))?;
+
+                            // Ensure ID exists - generate if missing
+                            if let Some(obj) = value.as_object_mut() {
+                                if !obj.contains_key("id")
+                                    || obj
+                                        .get("id")
+                                        .and_then(|v| v.as_str())
+                                        .map(|s| s.is_empty())
+                                        .unwrap_or(true)
+                                {
+                                    // Generate ID with appropriate prefix based on type
+                                    let item_type =
+                                        obj.get("type").and_then(|v| v.as_str()).unwrap_or("item");
+                                    let prefix = match item_type {
+                                        "function_call" | "function_call_output" => "fc",
+                                        "message" => "msg",
+                                        _ => "item",
+                                    };
+                                    obj.insert("id".to_string(), json!(generate_id(prefix)));
+                                }
+                            }
+
+                            Ok(value)
+                        }
+                    }
+                })
+                .collect::<Result<Vec<_>, String>>()?
+        }
+    };
+
+    Ok(items)
+}
+
+/// Convert a JSON item to NewConversationItem
+///
+/// For input items: function_call/function_call_output store whole item as content
+/// For output items: message extracts content field, others store whole item
+fn item_to_new_conversation_item(
+    item_value: &Value,
+    response_id: Option<String>,
+    is_input: bool,
+) -> NewConversationItem {
+    let item_type = item_value
+        .get("type")
+        .and_then(|v| v.as_str())
+        .unwrap_or("message");
+
+    // Determine if we should store the whole item or just the content field
+    let store_whole_item = if is_input {
+        item_type == "function_call" || item_type == "function_call_output"
+    } else {
+        item_type != "message"
+    };
+
+    let content = if store_whole_item {
+        item_value.clone()
+    } else {
+        item_value.get("content").cloned().unwrap_or(json!([]))
+    };
+
+    NewConversationItem {
+        id: item_value
+            .get("id")
+            .and_then(|v| v.as_str())
+            .map(ConversationItemId::from),
+        response_id,
+        item_type: item_type.to_string(),
+        role: item_value
+            .get("role")
+            .and_then(|v| v.as_str())
+            .map(String::from),
+        content,
+        status: item_value
+            .get("status")
+            .and_then(|v| v.as_str())
+            .map(String::from),
+    }
+}
+
+/// Link all input and output items to a conversation
+async fn link_items_to_conversation(
+    item_storage: &Arc<dyn ConversationItemStorage>,
+    conv_id: &ConversationId,
+    input_items: &[Value],
+    output_items: &[Value],
+    response_id: &str,
+) -> Result<(), String> {
+    let response_id_opt = Some(response_id.to_string());
+
+    for item in input_items {
+        let new_item = item_to_new_conversation_item(item, response_id_opt.clone(), true);
+        create_and_link_item(item_storage, Some(conv_id), new_item).await?;
+    }
+
+    for item in output_items {
+        let new_item = item_to_new_conversation_item(item, response_id_opt.clone(), false);
+        create_and_link_item(item_storage, Some(conv_id), new_item).await?;
+    }
+
+    Ok(())
+}
+
+/// Persist conversation items to storage
+///
+/// This function:
+/// 1. Extracts and normalizes input items from the request
+/// 2. Extracts output items from the response
+/// 3. Stores ALL items in response storage (always)
+/// 4. If conversation provided, also links items to conversation
+pub async fn persist_conversation_items(
+    conversation_storage: Arc<dyn ConversationStorage>,
+    item_storage: Arc<dyn ConversationItemStorage>,
+    response_storage: Arc<dyn ResponseStorage>,
+    response_json: &Value,
+    original_body: &ResponsesRequest,
+) -> Result<(), String> {
+    // Extract response ID
+    let response_id_str = response_json
+        .get("id")
+        .and_then(|v| v.as_str())
+        .ok_or_else(|| "Response missing id field".to_string())?;
+    let response_id = ResponseId::from(response_id_str);
+
+    // Parse and normalize input items from request
+    let input_items = extract_input_items(&original_body.input)?;
+
+    // Parse output items from response
+    let output_items = response_json
+        .get("output")
+        .and_then(|v| v.as_array())
+        .cloned()
+        .ok_or_else(|| "No output array in response".to_string())?;
+
+    // Build and store response
+    let mut stored_response = build_stored_response(response_json, original_body);
+    stored_response.id = response_id.clone();
+    stored_response.input = Value::Array(input_items.clone());
+    stored_response.output = Value::Array(output_items.clone());
+
+    response_storage
+        .store_response(stored_response)
+        .await
+        .map_err(|e| format!("Failed to store response: {}", e))?;
+
+    // Check if conversation is provided and validate it exists
+    let conv_id_opt = if let Some(id) = &original_body.conversation {
+        let conv_id = ConversationId::from(id.as_str());
+        match conversation_storage.get_conversation(&conv_id).await {
+            Ok(Some(_)) => Some(conv_id),
+            Ok(None) => {
+                warn!(conversation_id = %conv_id.0, "Conversation not found, skipping item linking");
+                None
+            }
+            Err(e) => return Err(format!("Failed to get conversation: {}", e)),
+        }
+    } else {
+        None
+    };
+
+    // If conversation exists, link items to it
+    if let Some(conv_id) = conv_id_opt {
+        link_items_to_conversation(
+            &item_storage,
+            &conv_id,
+            &input_items,
+            &output_items,
+            response_id_str,
+        )
+        .await?;
+        info!(
+            conversation_id = %conv_id.0,
+            response_id = %response_id.0,
+            input_count = input_items.len(),
+            output_count = output_items.len(),
+            "Persisted response and linked items to conversation"
+        );
+    } else {
+        info!(
+            response_id = %response_id.0,
+            input_count = input_items.len(),
+            output_count = output_items.len(),
+            "Persisted response without conversation linking"
+        );
+    }
+
+    Ok(())
+}

third_party/sglang/sgl-model-gateway/src/routers/router_manager.rs

@@ -0,0 +1,749 @@
+//! Router Manager for coordinating multiple routers and workers
+//!
+//! Provides centralized management based on enable_igw flag:
+//! - Single Router Mode (enable_igw=false): Router owns workers directly
+//! - Multi-Router Mode (enable_igw=true): RouterManager coordinates everything
+
+use std::sync::Arc;
+
+use arc_swap::ArcSwap;
+use async_trait::async_trait;
+use axum::{
+    body::Body,
+    extract::Request,
+    http::{HeaderMap, StatusCode},
+    response::{IntoResponse, Response},
+};
+use dashmap::DashMap;
+use serde_json::Value;
+use tracing::{debug, info, warn};
+
+use crate::{
+    app_context::AppContext,
+    config::RoutingMode,
+    core::{ConnectionMode, RuntimeType, WorkerRegistry, WorkerType},
+    protocols::{
+        chat::ChatCompletionRequest,
+        classify::ClassifyRequest,
+        completion::CompletionRequest,
+        embedding::EmbeddingRequest,
+        generate::GenerateRequest,
+        rerank::RerankRequest,
+        responses::{ResponsesGetParams, ResponsesRequest},
+    },
+    routers::RouterTrait,
+    server::ServerConfig,
+};
+
+#[derive(Debug, Clone, Hash, Eq, PartialEq)]
+pub struct RouterId(&'static str);
+
+impl RouterId {
+    pub const fn new(id: &'static str) -> Self {
+        Self(id)
+    }
+
+    pub fn as_str(&self) -> &str {
+        self.0
+    }
+}
+
+/// Static router ID constants to avoid heap allocations in hot paths
+pub mod router_ids {
+    use super::RouterId;
+
+    pub const HTTP_REGULAR: RouterId = RouterId::new("http-regular");
+    pub const HTTP_PD: RouterId = RouterId::new("http-pd");
+    pub const HTTP_OPENAI: RouterId = RouterId::new("http-openai");
+    pub const GRPC_REGULAR: RouterId = RouterId::new("grpc-regular");
+    pub const GRPC_PD: RouterId = RouterId::new("grpc-pd");
+}
+
+pub struct RouterManager {
+    worker_registry: Arc<WorkerRegistry>,
+    routers: Arc<DashMap<RouterId, Arc<dyn RouterTrait>>>,
+    routers_snapshot: ArcSwap<Vec<Arc<dyn RouterTrait>>>,
+    default_router: Arc<std::sync::RwLock<Option<RouterId>>>,
+    enable_igw: bool,
+}
+
+impl RouterManager {
+    pub fn new(worker_registry: Arc<WorkerRegistry>) -> Self {
+        Self {
+            worker_registry,
+            routers: Arc::new(DashMap::new()),
+            routers_snapshot: ArcSwap::from_pointee(Vec::new()),
+            default_router: Arc::new(std::sync::RwLock::new(None)),
+            enable_igw: false, // Will be set properly in from_config
+        }
+    }
+
+    pub async fn from_config(
+        config: &ServerConfig,
+        app_context: &Arc<AppContext>,
+    ) -> Result<Arc<Self>, String> {
+        use crate::routers::RouterFactory;
+
+        let mut manager = Self::new(app_context.worker_registry.clone());
+        manager.enable_igw = config.router_config.enable_igw;
+        let manager = Arc::new(manager);
+
+        if config.router_config.enable_igw {
+            info!("Initializing RouterManager in multi-router mode (IGW)");
+
+            match RouterFactory::create_regular_router(app_context).await {
+                Ok(http_regular) => {
+                    info!("Created HTTP Regular router");
+                    manager.register_router(router_ids::HTTP_REGULAR, Arc::from(http_regular));
+                }
+                Err(e) => {
+                    warn!("Failed to create HTTP Regular router: {e}");
+                }
+            }
+
+            // Always create gRPC Regular router in IGW mode
+            match RouterFactory::create_grpc_router(app_context).await {
+                Ok(grpc_regular) => {
+                    info!("Created gRPC Regular router");
+                    manager.register_router(router_ids::GRPC_REGULAR, Arc::from(grpc_regular));
+                }
+                Err(e) => {
+                    warn!("Failed to create gRPC Regular router: {e}");
+                }
+            }
+
+            info!("PD disaggregation auto-enabled for IGW mode, creating PD routers");
+
+            // Create HTTP PD router
+            match RouterFactory::create_pd_router(
+                None,
+                None,
+                &config.router_config.policy,
+                app_context,
+            )
+            .await
+            {
+                Ok(http_pd) => {
+                    info!("Created HTTP PD router");
+                    manager.register_router(router_ids::HTTP_PD, Arc::from(http_pd));
+                }
+                Err(e) => {
+                    warn!("Failed to create HTTP PD router: {e}");
+                }
+            }
+
+            // Create gRPC PD router
+            match RouterFactory::create_grpc_pd_router(
+                None,
+                None,
+                &config.router_config.policy,
+                app_context,
+            )
+            .await
+            {
+                Ok(grpc_pd) => {
+                    info!("Created gRPC PD router");
+                    manager.register_router(router_ids::GRPC_PD, Arc::from(grpc_pd));
+                }
+                Err(e) => {
+                    warn!("Failed to create gRPC PD router: {e}");
+                }
+            }
+
+            // Create OpenAI router for external OpenAI-compatible backends
+            match RouterFactory::create_openai_router(app_context).await {
+                Ok(openai) => {
+                    info!("Created OpenAI router");
+                    manager.register_router(router_ids::HTTP_OPENAI, Arc::from(openai));
+                }
+                Err(e) => {
+                    warn!("Failed to create OpenAI router: {e}");
+                }
+            }
+
+            info!(
+                "RouterManager initialized with {} routers for multi-router mode",
+                manager.router_count(),
+            );
+        } else {
+            info!("Initializing RouterManager in single-router mode");
+
+            let single_router = Arc::from(RouterFactory::create_router(app_context).await?);
+            let router_id = Self::determine_router_id(
+                &config.router_config.mode,
+                &config.router_config.connection_mode,
+            );
+
+            info!("Created single router with ID: {}", router_id.as_str());
+            manager.register_router(router_id.clone(), single_router);
+            manager.set_default_router(router_id);
+        }
+
+        if manager.router_count() == 0 {
+            return Err("No routers could be initialized".to_string());
+        }
+
+        Ok(manager)
+    }
+
+    pub fn determine_router_id(
+        routing_mode: &RoutingMode,
+        connection_mode: &ConnectionMode,
+    ) -> RouterId {
+        match (connection_mode, routing_mode) {
+            (ConnectionMode::Http, RoutingMode::Regular { .. }) => router_ids::HTTP_REGULAR,
+            (ConnectionMode::Http, RoutingMode::PrefillDecode { .. }) => router_ids::HTTP_PD,
+            (ConnectionMode::Http, RoutingMode::OpenAI { .. }) => router_ids::HTTP_OPENAI,
+            (ConnectionMode::Grpc { .. }, RoutingMode::Regular { .. }) => router_ids::GRPC_REGULAR,
+            (ConnectionMode::Grpc { .. }, RoutingMode::PrefillDecode { .. }) => router_ids::GRPC_PD,
+            (ConnectionMode::Grpc { .. }, RoutingMode::OpenAI { .. }) => router_ids::GRPC_REGULAR,
+        }
+    }
+
+    pub fn register_router(&self, id: RouterId, router: Arc<dyn RouterTrait>) {
+        self.routers.insert(id.clone(), router);
+
+        // Update the lock-free snapshot for fast per-request iteration
+        let new_snapshot: Vec<_> = self.routers.iter().map(|e| e.value().clone()).collect();
+        self.routers_snapshot.store(Arc::new(new_snapshot));
+
+        let mut default_router = self
+            .default_router
+            .write()
+            .unwrap_or_else(|e| e.into_inner());
+        if default_router.is_none() {
+            *default_router = Some(id.clone());
+            info!("Set default router to {}", id.as_str());
+        }
+    }
+
+    pub fn set_default_router(&self, id: RouterId) {
+        let mut default_router = self
+            .default_router
+            .write()
+            .unwrap_or_else(|e| e.into_inner());
+        *default_router = Some(id);
+    }
+
+    pub fn router_count(&self) -> usize {
+        self.routers.len()
+    }
+
+    /// Resolve model_id for a request, inferring from available workers if not specified.
+    ///
+    /// Behavior in IGW mode (must fail fast if model not resolvable):
+    /// - If model_id is provided, use it directly
+    /// - If not provided and only one model exists, use it as implicit default
+    /// - If not provided and multiple models exist, return error requiring specification
+    /// - If no models exist, return service unavailable error
+    fn resolve_model_id(&self, model_id: Option<&str>) -> Result<String, Box<Response>> {
+        // If model_id is provided, use it
+        if let Some(id) = model_id {
+            return Ok(id.to_string());
+        }
+
+        // Get all available models from worker registry
+        let available_models = self.worker_registry.get_models();
+
+        match available_models.len() {
+            0 => Err(Box::new(
+                (
+                    StatusCode::SERVICE_UNAVAILABLE,
+                    "No models available - no workers registered",
+                )
+                    .into_response(),
+            )),
+            1 => {
+                // Single model: use it as implicit default
+                debug!(
+                    "Model not specified, using implicit default: {}",
+                    available_models[0]
+                );
+                Ok(available_models[0].clone())
+            }
+            _ => {
+                // Multiple models: require explicit model specification
+                Err(Box::new(
+                    (
+                        StatusCode::BAD_REQUEST,
+                        format!(
+                            "Model must be specified. Available models: {}",
+                            available_models.join(", ")
+                        ),
+                    )
+                        .into_response(),
+                ))
+            }
+        }
+    }
+
+    pub fn get_router_for_model(&self, model_id: &str) -> Option<Arc<dyn RouterTrait>> {
+        let workers = self.worker_registry.get_by_model(model_id);
+
+        // Find the best router ID based on worker capabilities
+        // Priority: external (OpenAI) > grpc-pd > http-pd > grpc-regular > http-regular
+        let best_router_id = workers
+            .iter()
+            .map(|w| {
+                let is_pd = matches!(
+                    w.worker_type(),
+                    WorkerType::Prefill { .. } | WorkerType::Decode
+                );
+                let is_grpc = matches!(w.connection_mode(), ConnectionMode::Grpc { .. });
+                let is_external = matches!(w.metadata().runtime_type, RuntimeType::External);
+
+                if is_external {
+                    // External workers should be routed via OpenAI-compatible router
+                    return (4, &router_ids::HTTP_OPENAI);
+                }
+
+                match (is_grpc, is_pd) {
+                    (true, true) => (3, &router_ids::GRPC_PD),
+                    (false, true) => (2, &router_ids::HTTP_PD),
+                    (true, false) => (1, &router_ids::GRPC_REGULAR),
+                    (false, false) => (0, &router_ids::HTTP_REGULAR),
+                }
+            })
+            .max_by_key(|(score, _)| *score)
+            .map(|(_, id)| id);
+
+        if let Some(router_id) = best_router_id {
+            if let Some(router) = self.routers.get(router_id) {
+                return Some(router.clone());
+            }
+        }
+
+        // Fallback to default router
+        let default_router = self
+            .default_router
+            .read()
+            .unwrap_or_else(|e| e.into_inner());
+        if let Some(ref default_id) = *default_router {
+            self.routers.get(default_id).map(|r| r.clone())
+        } else {
+            None
+        }
+    }
+
+    pub fn select_router_for_request(
+        &self,
+        headers: Option<&HeaderMap>,
+        model_id: Option<&str>,
+    ) -> Option<Arc<dyn RouterTrait>> {
+        // In single-router mode (enable_igw=false), always use the default router
+        if !self.enable_igw {
+            let default_router = self
+                .default_router
+                .read()
+                .unwrap_or_else(|e| e.into_inner());
+            if let Some(ref default_id) = *default_router {
+                debug!(
+                    "Single-router mode: using default router {} for model {:?}",
+                    default_id.as_str(),
+                    model_id
+                );
+                return self.routers.get(default_id).map(|r| r.clone());
+            }
+        }
+
+        let prefer_pd = headers
+            .and_then(|h| {
+                h.get("x-prefer-pd")
+                    .and_then(|v| v.to_str().ok())
+                    .map(|s| s == "true" || s == "1")
+            })
+            .unwrap_or(false);
+
+        let (num_regular_workers, num_pd_workers) = self.worker_registry.get_worker_distribution();
+        let mut best_router = None;
+        let mut best_score = -1.0;
+
+        // Extract router validity check into a closure to reduce redundancy
+        let is_router_valid =
+            |is_pd: bool| (is_pd && num_pd_workers > 0) || (!is_pd && num_regular_workers > 0);
+
+        if let Some(model) = model_id {
+            // Efficient Single Lookup for Specific Model
+            if let Some(router) = self.get_router_for_model(model) {
+                if is_router_valid(router.is_pd_mode()) {
+                    return Some(router);
+                }
+            }
+        } else {
+            // ZERO-ALLOCATION Snapshot Iteration (Hot Path Optimization)
+            // Atomic load avoids heap allocations and DashMap shard locks per-request
+            let routers_snapshot = self.routers_snapshot.load();
+            for router in routers_snapshot.iter() {
+                let mut score = 1.0;
+
+                let is_pd = router.is_pd_mode();
+                if prefer_pd && is_pd {
+                    score += 2.0;
+                } else if !prefer_pd && !is_pd {
+                    score += 1.0;
+                }
+                // TODO: Once routers expose worker stats, we can evaluate:
+                // - Average worker priority vs priority_threshold
+                // - Average worker cost vs max_cost
+                // - Current load and health status
+
+                if score > best_score && is_router_valid(is_pd) {
+                    best_score = score;
+                    best_router = Some(Arc::clone(router));
+                }
+            }
+        }
+
+        best_router
+    }
+}
+
+#[async_trait]
+impl RouterTrait for RouterManager {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    async fn health_generate(&self, _req: Request<Body>) -> Response {
+        // IGW readiness: return 200 if at least one router has healthy workers
+        let has_healthy_workers = self
+            .worker_registry
+            .get_all()
+            .iter()
+            .any(|w| w.is_healthy());
+
+        if has_healthy_workers {
+            (StatusCode::OK, "At least one router has healthy workers").into_response()
+        } else {
+            (
+                StatusCode::SERVICE_UNAVAILABLE,
+                "No routers with healthy workers available",
+            )
+                .into_response()
+        }
+    }
+
+    async fn get_server_info(&self, _req: Request<Body>) -> Response {
+        // TODO: Aggregate info from all routers with healthy workers
+        (
+            StatusCode::OK,
+            serde_json::json!({
+                "router_manager": true,
+                "routers_count": self.routers.len(),
+                "workers_count": self.worker_registry.get_all().len()
+            })
+            .to_string(),
+        )
+            .into_response()
+    }
+
+    async fn get_models(&self, _req: Request<Body>) -> Response {
+        let model_names = self.worker_registry.get_models();
+
+        if model_names.is_empty() {
+            (StatusCode::SERVICE_UNAVAILABLE, "No models available").into_response()
+        } else {
+            // Convert model names to OpenAI-compatible model objects
+            let models: Vec<Value> = model_names
+                .iter()
+                .map(|name| {
+                    serde_json::json!({
+                        "id": name,
+                        "object": "model",
+                        "owned_by": "local"
+                    })
+                })
+                .collect();
+
+            (
+                StatusCode::OK,
+                serde_json::json!({
+                    "object": "list",
+                    "data": models
+                })
+                .to_string(),
+            )
+                .into_response()
+        }
+    }
+
+    async fn get_model_info(&self, req: Request<Body>) -> Response {
+        // Route to default router or first available router
+        let router_id = {
+            let default_router = self
+                .default_router
+                .read()
+                .unwrap_or_else(|e| e.into_inner());
+            default_router.clone()
+        };
+
+        let router = if let Some(id) = router_id {
+            self.routers.get(&id).map(|r| r.clone())
+        } else {
+            // If no default, use first available router
+            self.routers.iter().next().map(|r| r.value().clone())
+        };
+
+        if let Some(router) = router {
+            router.get_model_info(req).await
+        } else {
+            (StatusCode::SERVICE_UNAVAILABLE, "No routers available").into_response()
+        }
+    }
+
+    async fn route_generate(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &GenerateRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        // In IGW mode, resolve model_id and fail fast if not resolvable
+        // In non-IGW mode, pass through to router (router handles validation)
+        let effective_model_id = if self.enable_igw {
+            match self.resolve_model_id(model_id) {
+                Ok(id) => Some(id),
+                Err(err_response) => return *err_response,
+            }
+        } else {
+            None
+        };
+
+        let router =
+            self.select_router_for_request(headers, effective_model_id.as_deref().or(model_id));
+
+        if let Some(router) = router {
+            router
+                .route_generate(headers, body, effective_model_id.as_deref().or(model_id))
+                .await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                "No router available for this request",
+            )
+                .into_response()
+        }
+    }
+
+    async fn route_chat(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ChatCompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        // In IGW mode, resolve model_id and fail fast if not resolvable
+        // In non-IGW mode, pass through to router (router handles validation)
+        let effective_model_id = if self.enable_igw {
+            // Use provided model_id or fall back to body.model
+            let model = model_id.or(Some(&body.model));
+            match self.resolve_model_id(model) {
+                Ok(id) => Some(id),
+                Err(err_response) => return *err_response,
+            }
+        } else {
+            None
+        };
+
+        let router =
+            self.select_router_for_request(headers, effective_model_id.as_deref().or(model_id));
+
+        if let Some(router) = router {
+            router
+                .route_chat(headers, body, effective_model_id.as_deref().or(model_id))
+                .await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                format!("Model '{}' not found or no router available", body.model),
+            )
+                .into_response()
+        }
+    }
+
+    async fn route_completion(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &CompletionRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        // In IGW mode, resolve model_id and fail fast if not resolvable
+        // In non-IGW mode, pass through to router (router handles validation)
+        let effective_model_id = if self.enable_igw {
+            // Use provided model_id or fall back to body.model
+            let model = model_id.or(Some(&body.model));
+            match self.resolve_model_id(model) {
+                Ok(id) => Some(id),
+                Err(err_response) => return *err_response,
+            }
+        } else {
+            None
+        };
+
+        let router =
+            self.select_router_for_request(headers, effective_model_id.as_deref().or(model_id));
+
+        if let Some(router) = router {
+            router
+                .route_completion(headers, body, effective_model_id.as_deref().or(model_id))
+                .await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                format!("Model '{}' not found or no router available", body.model),
+            )
+                .into_response()
+        }
+    }
+
+    async fn route_responses(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ResponsesRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        let selected_model = model_id.or(Some(body.model.as_str()));
+        let router = self.select_router_for_request(headers, selected_model);
+
+        if let Some(router) = router {
+            router.route_responses(headers, body, selected_model).await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                "No router available to handle responses request",
+            )
+                .into_response()
+        }
+    }
+
+    async fn get_response(
+        &self,
+        headers: Option<&HeaderMap>,
+        response_id: &str,
+        params: &ResponsesGetParams,
+    ) -> Response {
+        let router = self.select_router_for_request(headers, None);
+        if let Some(router) = router {
+            router.get_response(headers, response_id, params).await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                format!("No router available to get response '{}'", response_id),
+            )
+                .into_response()
+        }
+    }
+
+    async fn cancel_response(&self, headers: Option<&HeaderMap>, response_id: &str) -> Response {
+        let router = self.select_router_for_request(headers, None);
+        if let Some(router) = router {
+            router.cancel_response(headers, response_id).await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                format!("No router available to cancel response '{}'", response_id),
+            )
+                .into_response()
+        }
+    }
+
+    async fn delete_response(&self, _headers: Option<&HeaderMap>, _response_id: &str) -> Response {
+        (
+            StatusCode::NOT_IMPLEMENTED,
+            "responses api not yet implemented in inference gateway mode",
+        )
+            .into_response()
+    }
+
+    async fn list_response_input_items(
+        &self,
+        headers: Option<&HeaderMap>,
+        response_id: &str,
+    ) -> Response {
+        // Delegate to the default router (typically http-regular)
+        // Response storage is shared across all routers via AppContext
+        let router = self.select_router_for_request(headers, None);
+        if let Some(router) = router {
+            router.list_response_input_items(headers, response_id).await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                "No router available to list response input items",
+            )
+                .into_response()
+        }
+    }
+
+    async fn route_embeddings(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &EmbeddingRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        let router = self.select_router_for_request(headers, model_id);
+
+        if let Some(router) = router {
+            router.route_embeddings(headers, body, model_id).await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                format!("Model '{}' not found or no router available", body.model),
+            )
+                .into_response()
+        }
+    }
+
+    async fn route_classify(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &ClassifyRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        let router = self.select_router_for_request(headers, model_id);
+
+        if let Some(router) = router {
+            router.route_classify(headers, body, model_id).await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                format!("Model '{}' not found or no router available", body.model),
+            )
+                .into_response()
+        }
+    }
+
+    async fn route_rerank(
+        &self,
+        headers: Option<&HeaderMap>,
+        body: &RerankRequest,
+        model_id: Option<&str>,
+    ) -> Response {
+        let router = self.select_router_for_request(headers, model_id);
+
+        if let Some(router) = router {
+            router.route_rerank(headers, body, model_id).await
+        } else {
+            (
+                StatusCode::NOT_FOUND,
+                "No router available for rerank request",
+            )
+                .into_response()
+        }
+    }
+
+    fn router_type(&self) -> &'static str {
+        "manager"
+    }
+}
+
+impl std::fmt::Debug for RouterManager {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let default_router = self
+            .default_router
+            .read()
+            .unwrap_or_else(|e| e.into_inner());
+        f.debug_struct("RouterManager")
+            .field("routers_count", &self.routers.len())
+            .field("workers_count", &self.worker_registry.get_all().len())
+            .field("default_router", &*default_router)
+            .finish()
+    }
+}

third_party/sglang/sgl-model-gateway/src/routers/tokenize/handlers.rs

@@ -0,0 +1,394 @@
+//! Tokenize and detokenize handlers
+//!
+//! Provides tokenization, detokenization, and tokenizer management operations.
+//! These handlers use the TokenizerRegistry for tokenizer storage and retrieval.
+
+use std::sync::Arc;
+
+use axum::{
+    http::StatusCode,
+    response::{IntoResponse, Response},
+    Json,
+};
+use tracing::{debug, error, warn};
+
+use crate::{
+    app_context::AppContext,
+    core::{steps::TokenizerConfigRequest, Job, UNKNOWN_MODEL_ID},
+    protocols::tokenize::{
+        AddTokenizerRequest, AddTokenizerResponse, CountResult, DetokenizeRequest,
+        DetokenizeResponse, ListTokenizersResponse, RemoveTokenizerResponse, TextResult,
+        TokenizeRequest, TokenizeResponse, TokenizerInfo, TokensResult,
+    },
+    tokenizer::{registry::TokenizerEntry, traits::Tokenizer, TokenizerRegistry},
+};
+
+/// Helper to create error responses
+fn error_response(status: StatusCode, message: &str, error_type: &str) -> Response {
+    (
+        status,
+        Json(serde_json::json!({
+            "error": {
+                "message": message,
+                "type": error_type
+            }
+        })),
+    )
+        .into_response()
+}
+
+/// Get a tokenizer by model name, with fallback strategies
+fn get_tokenizer(registry: &TokenizerRegistry, model: &str) -> Result<Arc<dyn Tokenizer>, String> {
+    // First, try exact match (by name or ID)
+    if let Some(tokenizer) = registry.get(model) {
+        debug!("Found tokenizer for model: {}", model);
+        return Ok(tokenizer);
+    }
+
+    // Try UNKNOWN_MODEL_ID if model is "unknown" or empty
+    if model == UNKNOWN_MODEL_ID || model.is_empty() {
+        // Try to find any tokenizer as fallback
+        let entries = registry.list();
+        if let Some(first) = entries.first() {
+            debug!(
+                "Using first available tokenizer '{}' as default",
+                first.name
+            );
+            return Ok(first.tokenizer.clone());
+        }
+    }
+
+    // List available tokenizers for error message
+    let entries = registry.list();
+    if entries.is_empty() {
+        Err("No tokenizers available. Use POST /v1/tokenizers to add one.".to_string())
+    } else {
+        let names: Vec<&str> = entries.iter().map(|e| e.name.as_str()).collect();
+        Err(format!(
+            "Tokenizer for model '{}' not found. Available: {}",
+            model,
+            names.join(", ")
+        ))
+    }
+}
+
+// ============================================================================
+// Tokenize / Detokenize Handlers
+// ============================================================================
+
+/// Handle POST /v1/tokenize
+pub async fn tokenize(registry: &Arc<TokenizerRegistry>, request: TokenizeRequest) -> Response {
+    debug!("Tokenize request for model: {}", request.model);
+
+    let tokenizer = match get_tokenizer(registry, &request.model) {
+        Ok(t) => t,
+        Err(e) => {
+            return error_response(StatusCode::BAD_REQUEST, &e, "tokenizer_not_found");
+        }
+    };
+
+    let texts = request.prompt.as_strings();
+    let is_batch = request.prompt.is_batch();
+
+    // Tokenize each text
+    let mut all_tokens: Vec<Vec<u32>> = Vec::with_capacity(texts.len());
+    let mut all_counts: Vec<i32> = Vec::with_capacity(texts.len());
+    let mut all_char_counts: Vec<i32> = Vec::with_capacity(texts.len());
+
+    for text in texts {
+        // Don't add special tokens for tokenize API (matches Python behavior)
+        let encoding = match tokenizer.encode(text, false) {
+            Ok(enc) => enc,
+            Err(e) => {
+                error!("Tokenization failed: {}", e);
+                return error_response(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    &format!("Tokenization failed: {}", e),
+                    "tokenization_error",
+                );
+            }
+        };
+
+        let token_ids: Vec<u32> = encoding.token_ids().to_vec();
+        let count = token_ids.len() as i32;
+
+        all_tokens.push(token_ids);
+        all_counts.push(count);
+        all_char_counts.push(text.chars().count() as i32);
+    }
+
+    // Format response based on single vs batch
+    let (tokens, count, char_count) = if is_batch {
+        (
+            TokensResult::Batch(all_tokens),
+            CountResult::Batch(all_counts),
+            CountResult::Batch(all_char_counts),
+        )
+    } else {
+        (
+            TokensResult::Single(all_tokens.into_iter().next().unwrap_or_default()),
+            CountResult::Single(all_counts.into_iter().next().unwrap_or(0)),
+            CountResult::Single(all_char_counts.into_iter().next().unwrap_or(0)),
+        )
+    };
+
+    Json(TokenizeResponse {
+        tokens,
+        count,
+        char_count,
+    })
+    .into_response()
+}
+
+/// Handle POST /v1/detokenize
+pub async fn detokenize(registry: &Arc<TokenizerRegistry>, request: DetokenizeRequest) -> Response {
+    debug!("Detokenize request for model: {}", request.model);
+
+    let tokenizer = match get_tokenizer(registry, &request.model) {
+        Ok(t) => t,
+        Err(e) => {
+            return error_response(StatusCode::BAD_REQUEST, &e, "tokenizer_not_found");
+        }
+    };
+
+    let sequences = request.tokens.sequences();
+    let is_batch = request.tokens.is_batch();
+
+    // Detokenize each sequence
+    let mut all_texts: Vec<String> = Vec::with_capacity(sequences.len());
+
+    for seq in sequences {
+        let text = match tokenizer.decode(seq, request.skip_special_tokens) {
+            Ok(t) => t,
+            Err(e) => {
+                error!("Detokenization failed: {}", e);
+                return error_response(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    &format!("Detokenization failed: {}", e),
+                    "detokenization_error",
+                );
+            }
+        };
+        all_texts.push(text);
+    }
+
+    // Format response based on single vs batch
+    let text = if is_batch {
+        TextResult::Batch(all_texts)
+    } else {
+        TextResult::Single(all_texts.into_iter().next().unwrap_or_default())
+    };
+
+    Json(DetokenizeResponse { text }).into_response()
+}
+
+// ============================================================================
+// Tokenizer Management Handlers
+// ============================================================================
+
+/// Handle POST /v1/tokenizers - async version using job queue
+pub async fn add_tokenizer(context: &Arc<AppContext>, request: AddTokenizerRequest) -> Response {
+    // Check if tokenizer already exists by name
+    if context.tokenizer_registry.contains(&request.name) {
+        // Return the existing tokenizer's ID
+        if let Some(entry) = context.tokenizer_registry.get_by_name(&request.name) {
+            return (
+                StatusCode::CONFLICT,
+                Json(AddTokenizerResponse {
+                    id: entry.id,
+                    status: "failed".to_string(),
+                    message: format!("Tokenizer '{}' already exists", request.name),
+                    vocab_size: None,
+                }),
+            )
+                .into_response();
+        }
+    }
+
+    // Get the job queue
+    let job_queue = match context.worker_job_queue.get() {
+        Some(queue) => queue,
+        None => {
+            error!("Job queue not available");
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(AddTokenizerResponse {
+                    id: String::new(),
+                    status: "failed".to_string(),
+                    message: "Job queue not available".to_string(),
+                    vocab_size: None,
+                }),
+            )
+                .into_response();
+        }
+    };
+
+    // Generate UUID for this tokenizer
+    let tokenizer_id = TokenizerRegistry::generate_id();
+
+    // Create the job with the pre-generated ID
+    // Note: API-initiated tokenizer loads don't use caching by default
+    // Caching is applied for startup and worker-initiated loads based on router config
+    let config = TokenizerConfigRequest {
+        id: tokenizer_id.clone(),
+        name: request.name.clone(),
+        source: request.source.clone(),
+        chat_template_path: request.chat_template_path.clone(),
+        cache_config: None,
+        fail_on_duplicate: true,
+    };
+
+    let job = Job::AddTokenizer {
+        config: Box::new(config),
+    };
+
+    // Submit the job
+    match job_queue.submit(job).await {
+        Ok(()) => (
+            StatusCode::ACCEPTED,
+            Json(AddTokenizerResponse {
+                id: tokenizer_id,
+                status: "pending".to_string(),
+                message: format!(
+                    "Tokenizer '{}' registration job submitted. Loading from: {}",
+                    request.name, request.source
+                ),
+                vocab_size: None,
+            }),
+        )
+            .into_response(),
+        Err(e) => {
+            error!("Failed to submit tokenizer job: {}", e);
+            (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(AddTokenizerResponse {
+                    id: String::new(),
+                    status: "failed".to_string(),
+                    message: e,
+                    vocab_size: None,
+                }),
+            )
+                .into_response()
+        }
+    }
+}
+
+/// Handle GET /v1/tokenizers
+pub async fn list_tokenizers(registry: &Arc<TokenizerRegistry>) -> Response {
+    debug!("List tokenizers request");
+
+    let entries = registry.list();
+    let tokenizers: Vec<TokenizerInfo> = entries
+        .into_iter()
+        .map(|e| TokenizerInfo {
+            id: e.id,
+            name: e.name,
+            source: e.source,
+            vocab_size: e.tokenizer.vocab_size(),
+        })
+        .collect();
+
+    Json(ListTokenizersResponse { tokenizers }).into_response()
+}
+
+/// Handle DELETE /v1/tokenizers/{tokenizer_id}
+pub async fn remove_tokenizer(context: &Arc<AppContext>, tokenizer_id: &str) -> Response {
+    // Try to remove by ID first, then by name for backward compatibility
+    let removed = context
+        .tokenizer_registry
+        .remove_by_id(tokenizer_id)
+        .or_else(|| context.tokenizer_registry.remove(tokenizer_id));
+
+    if let Some(entry) = removed {
+        debug!("Removed tokenizer '{}' (id: {})", entry.name, entry.id);
+        (
+            StatusCode::OK,
+            Json(RemoveTokenizerResponse {
+                success: true,
+                message: format!("Tokenizer '{}' removed successfully", entry.name),
+            }),
+        )
+            .into_response()
+    } else {
+        warn!("Tokenizer '{}' not found", tokenizer_id);
+        (
+            StatusCode::NOT_FOUND,
+            Json(RemoveTokenizerResponse {
+                success: false,
+                message: format!("Tokenizer '{}' not found", tokenizer_id),
+            }),
+        )
+            .into_response()
+    }
+}
+
+/// Handle GET /v1/tokenizers/{tokenizer_id}
+pub async fn get_tokenizer_info(context: &Arc<AppContext>, tokenizer_id: &str) -> Response {
+    debug!("Get tokenizer info for '{}'", tokenizer_id);
+
+    // Try by ID first, then by name
+    let entry: Option<TokenizerEntry> = context
+        .tokenizer_registry
+        .get_by_id(tokenizer_id)
+        .or_else(|| context.tokenizer_registry.get_by_name(tokenizer_id));
+
+    match entry {
+        Some(e) => {
+            let info = TokenizerInfo {
+                id: e.id,
+                name: e.name,
+                source: e.source,
+                vocab_size: e.tokenizer.vocab_size(),
+            };
+            Json(info).into_response()
+        }
+        None => error_response(
+            StatusCode::NOT_FOUND,
+            &format!("Tokenizer '{}' not found", tokenizer_id),
+            "tokenizer_not_found",
+        ),
+    }
+}
+
+/// Handle GET /v1/tokenizers/{tokenizer_id}/status
+pub async fn get_tokenizer_status(context: &Arc<AppContext>, tokenizer_id: &str) -> Response {
+    debug!("Get tokenizer status for '{}'", tokenizer_id);
+
+    // First check if tokenizer is already loaded (by ID or name)
+    let entry = context
+        .tokenizer_registry
+        .get_by_id(tokenizer_id)
+        .or_else(|| context.tokenizer_registry.get_by_name(tokenizer_id));
+
+    if let Some(e) = entry {
+        return Json(AddTokenizerResponse {
+            id: e.id,
+            status: "completed".to_string(),
+            message: format!("Tokenizer '{}' is loaded and ready", e.name),
+            vocab_size: Some(e.tokenizer.vocab_size()),
+        })
+        .into_response();
+    }
+
+    // Check job status (jobs are tracked by ID)
+    if let Some(job_queue) = context.worker_job_queue.get() {
+        if let Some(job_status) = job_queue.get_status(tokenizer_id) {
+            return Json(AddTokenizerResponse {
+                id: tokenizer_id.to_string(),
+                status: job_status.status.clone(),
+                message: job_status
+                    .message
+                    .unwrap_or_else(|| format!("Tokenizer job is {}", job_status.status)),
+                vocab_size: None,
+            })
+            .into_response();
+        }
+    }
+
+    // Not found
+    error_response(
+        StatusCode::NOT_FOUND,
+        &format!("Tokenizer '{}' not found and no pending job", tokenizer_id),
+        "not_found",
+    )
+}

third_party/sglang/sgl-model-gateway/src/routers/tokenize/mod.rs

@@ -0,0 +1,13 @@
+//! Tokenize module for tokenization and detokenization operations
+//!
+//! This module provides HTTP handlers for:
+//! - Tokenizing text into token IDs
+//! - Detokenizing token IDs back to text
+//! - Managing tokenizers (add, list, get, remove)
+
+mod handlers;
+
+pub use handlers::{
+    add_tokenizer, detokenize, get_tokenizer_info, get_tokenizer_status, list_tokenizers,
+    remove_tokenizer, tokenize,
+};