chore: vendor sglang v0.5.10 snapshot
This commit is contained in:
239
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/Makefile
vendored
Normal file
239
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/Makefile
vendored
Normal file
@@ -0,0 +1,239 @@
|
||||
# Makefile for OAI Server
|
||||
# Builds binary, runs tests, and provides basic targets
|
||||
|
||||
# Configuration
|
||||
APP_NAME = oai_server
|
||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
|
||||
BUILD_TIME := $(shell date -u '+%Y-%m-%d_%H:%M:%S')
|
||||
GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
|
||||
# Paths
|
||||
ROOT_DIR := $(shell pwd)
|
||||
BINDINGS_DIR := $(shell cd $(ROOT_DIR)/../.. && pwd)
|
||||
BUILD_DIR := $(ROOT_DIR)/build
|
||||
BINARY := $(BUILD_DIR)/$(APP_NAME)
|
||||
|
||||
# Rust FFI library paths
|
||||
LIB_DIR := $(BINDINGS_DIR)/lib
|
||||
LIB_NAME = libsgl_model_gateway_go
|
||||
|
||||
# Detect OS
|
||||
UNAME_S := $(shell uname -s)
|
||||
ifeq ($(UNAME_S),Linux)
|
||||
LIB_EXT = .so
|
||||
LD_LIBRARY_PATH_VAR = LD_LIBRARY_PATH
|
||||
ARCH := $(shell uname -m)
|
||||
ifeq ($(ARCH),x86_64)
|
||||
GOARCH = amd64
|
||||
else ifeq ($(ARCH),aarch64)
|
||||
GOARCH = arm64
|
||||
endif
|
||||
endif
|
||||
ifeq ($(UNAME_S),Darwin)
|
||||
LIB_EXT = .dylib
|
||||
LD_LIBRARY_PATH_VAR = DYLD_LIBRARY_PATH
|
||||
ARCH := $(shell uname -m)
|
||||
ifeq ($(ARCH),x86_64)
|
||||
GOARCH = amd64
|
||||
else ifeq ($(ARCH),arm64)
|
||||
GOARCH = arm64
|
||||
endif
|
||||
endif
|
||||
|
||||
# Build flags
|
||||
LDFLAGS = -X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME) -X main.GitCommit=$(GIT_COMMIT)
|
||||
GO_BUILD_FLAGS = -ldflags "$(LDFLAGS)"
|
||||
|
||||
# Python LDFLAGS (needed for Rust FFI that depends on Python)
|
||||
PYTHON_LDFLAGS := $(shell python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || python-config --ldflags --embed 2>/dev/null || python-config --ldflags 2>/dev/null || echo "")
|
||||
|
||||
# CGO flags
|
||||
CGO_LDFLAGS = -L$(LIB_DIR) $(PYTHON_LDFLAGS)
|
||||
|
||||
.PHONY: all build build-dev test e2e clean help lib run stream check-rust-lib check-server
|
||||
|
||||
# E2E test configuration
|
||||
E2E_HOST ?= localhost
|
||||
E2E_PORT ?= 8080
|
||||
E2E_MODEL ?= default
|
||||
E2E_TOKENIZER ?= $(shell echo $$SGL_TOKENIZER_PATH || echo "./examples/tokenizer")
|
||||
E2E_NUM_PROMPTS ?= 100
|
||||
E2E_INPUT_LEN ?= 1024
|
||||
E2E_OUTPUT_LEN ?= 512
|
||||
E2E_REQUEST_RATE ?= 20
|
||||
E2E_MAX_CONCURRENCY ?= 20
|
||||
E2E_BASE_URL ?= http://$(E2E_HOST):$(E2E_PORT)
|
||||
|
||||
help:
|
||||
@echo "OAI Server Makefile"
|
||||
@echo ""
|
||||
@echo "Available targets:"
|
||||
@echo " lib - Build Rust FFI library"
|
||||
@echo " build - Build binary (release mode)"
|
||||
@echo " build-dev - Build binary (debug mode)"
|
||||
@echo " test - Run tests"
|
||||
@echo " e2e - Run end-to-end test with bench_serving.py"
|
||||
@echo " run - Run the server (development)"
|
||||
@echo " stream - Run streaming example"
|
||||
@echo " clean - Clean build artifacts"
|
||||
@echo ""
|
||||
@echo "E2E test variables:"
|
||||
@echo " E2E_HOST - OAI Server host (default: localhost)"
|
||||
@echo " E2E_PORT - OAI Server port (default: 8080)"
|
||||
@echo " E2E_MODEL - Model name (default: default)"
|
||||
@echo " E2E_TOKENIZER - Tokenizer path"
|
||||
@echo " E2E_NUM_PROMPTS - Number of prompts (default: 100)"
|
||||
@echo " E2E_INPUT_LEN - Input token length (default: 1024)"
|
||||
@echo " E2E_OUTPUT_LEN - Output token length (default: 512)"
|
||||
@echo " E2E_REQUEST_RATE - Request rate per second (default: 20)"
|
||||
@echo " E2E_MAX_CONCURRENCY - Max concurrent requests (default: 20)"
|
||||
|
||||
all: build
|
||||
|
||||
# Build Rust FFI library
|
||||
lib:
|
||||
@echo "Building Rust FFI library..."
|
||||
@cd $(BINDINGS_DIR) && $(MAKE) lib
|
||||
@echo "✓ Rust FFI library built"
|
||||
|
||||
# Check if Rust FFI library exists
|
||||
check-rust-lib:
|
||||
@if [ ! -f "$(LIB_DIR)/$(LIB_NAME)$(LIB_EXT)" ]; then \
|
||||
echo "Error: Rust FFI library not found at $(LIB_DIR)/$(LIB_NAME)$(LIB_EXT)"; \
|
||||
echo "Building Rust library..."; \
|
||||
cd $(BINDINGS_DIR) && $(MAKE) lib; \
|
||||
fi
|
||||
@echo "✓ Rust FFI library found"
|
||||
|
||||
# Build binary (release)
|
||||
build: check-rust-lib
|
||||
@echo "Building $(APP_NAME) (release mode)..."
|
||||
@mkdir -p $(BUILD_DIR)
|
||||
@CGO_ENABLED=1 \
|
||||
CGO_LDFLAGS="$(CGO_LDFLAGS)" \
|
||||
GOOS=$(shell go env GOOS) \
|
||||
GOARCH=$(GOARCH) \
|
||||
go build $(GO_BUILD_FLAGS) -o $(BINARY) .
|
||||
@echo "✓ Binary built: $(BINARY)"
|
||||
|
||||
# Build binary (debug)
|
||||
build-dev: check-rust-lib
|
||||
@echo "Building $(APP_NAME) (debug mode)..."
|
||||
@mkdir -p $(BUILD_DIR)
|
||||
@CGO_ENABLED=1 \
|
||||
CGO_LDFLAGS="$(CGO_LDFLAGS)" \
|
||||
go build -o $(BINARY) .
|
||||
@echo "✓ Binary built (debug): $(BINARY)"
|
||||
|
||||
# Run tests
|
||||
test: check-rust-lib
|
||||
@echo "Running tests..."
|
||||
@CGO_ENABLED=1 \
|
||||
CGO_LDFLAGS="$(CGO_LDFLAGS)" \
|
||||
export $(LD_LIBRARY_PATH_VAR)="$(LIB_DIR):$$$(LD_LIBRARY_PATH_VAR)" && \
|
||||
go test -v ./...
|
||||
@echo "✓ Tests completed"
|
||||
|
||||
# Check if OAI Server is running
|
||||
check-server:
|
||||
@echo "Checking if OAI Server is running at $(E2E_BASE_URL)..."
|
||||
@if curl -s -f $(E2E_BASE_URL)/health > /dev/null 2>&1; then \
|
||||
echo "✓ OAI Server is running"; \
|
||||
exit 0; \
|
||||
else \
|
||||
echo "✗ OAI Server is not running at $(E2E_BASE_URL)"; \
|
||||
echo " Start it with: make run"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Find sglang project root (4 levels up from oai_server)
|
||||
SGLANG_ROOT := $(shell cd $(ROOT_DIR)/../../../../.. && pwd)
|
||||
|
||||
# Run end-to-end test with bench_serving.py
|
||||
e2e: check-server
|
||||
@echo "Checking if bench_serving.py is available..."
|
||||
@if python -m sglang.bench_serving --help > /dev/null 2>&1; then \
|
||||
echo "✓ Using installed bench_serving.py module"; \
|
||||
USE_SGLANG_ROOT=false; \
|
||||
elif [ -f "$(SGLANG_ROOT)/python/sglang/bench_serving.py" ]; then \
|
||||
echo "✓ Using bench_serving.py from $(SGLANG_ROOT)"; \
|
||||
USE_SGLANG_ROOT=true; \
|
||||
else \
|
||||
echo "✗ bench_serving.py is not available"; \
|
||||
echo " Install dependencies: pip install aiohttp numpy datasets transformers tqdm pillow pybase64"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "Running end-to-end test with bench_serving.py..."
|
||||
@echo "Configuration:"
|
||||
@echo " Server: $(E2E_BASE_URL)"
|
||||
@if [ "$(E2E_MODEL)" != "default" ]; then \
|
||||
echo " Model: $(E2E_MODEL)"; \
|
||||
fi
|
||||
@if [ -n "$(E2E_TOKENIZER)" ]; then \
|
||||
echo " Tokenizer: $(E2E_TOKENIZER)"; \
|
||||
fi
|
||||
@echo " Prompts: $(E2E_NUM_PROMPTS)"
|
||||
@echo " Input/Output: $(E2E_INPUT_LEN)/$(E2E_OUTPUT_LEN) tokens"
|
||||
@echo " Request rate: $(E2E_REQUEST_RATE) req/s"
|
||||
@echo " Max concurrency: $(E2E_MAX_CONCURRENCY)"
|
||||
@echo ""
|
||||
@TOKENIZER_ABS=$$(cd $(ROOT_DIR) && python3 -c "import os; path='$(E2E_TOKENIZER)'; print(os.path.abspath(path) if not os.path.isabs(path) else path)" 2>/dev/null || echo "$(E2E_TOKENIZER)"); \
|
||||
if [ -n "$(E2E_TOKENIZER)" ]; then \
|
||||
if [ -n "$$TOKENIZER_ABS" ] && ([ -d "$$TOKENIZER_ABS" ] || [ -f "$$TOKENIZER_ABS" ]); then \
|
||||
TOKENIZER_ARG="--tokenizer $$TOKENIZER_ABS"; \
|
||||
else \
|
||||
TOKENIZER_ARG="--tokenizer $(E2E_TOKENIZER)"; \
|
||||
fi; \
|
||||
else \
|
||||
TOKENIZER_ARG=""; \
|
||||
fi; \
|
||||
if [ "$$USE_SGLANG_ROOT" = "true" ]; then \
|
||||
cd $(SGLANG_ROOT) && PYTHONPATH=$(SGLANG_ROOT)/python:$$PYTHONPATH python python/sglang/bench_serving.py \
|
||||
--backend sglang-oai-chat \
|
||||
--base-url $(E2E_BASE_URL) \
|
||||
$$([ "$(E2E_MODEL)" != "default" ] && echo "--model $(E2E_MODEL)") \
|
||||
$$TOKENIZER_ARG \
|
||||
--dataset-name random \
|
||||
--num-prompts $(E2E_NUM_PROMPTS) \
|
||||
--random-input-len $(E2E_INPUT_LEN) \
|
||||
--random-output-len $(E2E_OUTPUT_LEN) \
|
||||
--request-rate $(E2E_REQUEST_RATE) \
|
||||
--max-concurrency $(E2E_MAX_CONCURRENCY) \
|
||||
--warmup-requests 5 \
|
||||
--disable-tqdm || (echo "✗ E2E test failed"; exit 1); \
|
||||
else \
|
||||
python -m sglang.bench_serving \
|
||||
--backend sglang-oai-chat \
|
||||
--base-url $(E2E_BASE_URL) \
|
||||
$$([ "$(E2E_MODEL)" != "default" ] && echo "--model $(E2E_MODEL)") \
|
||||
$$TOKENIZER_ARG \
|
||||
--dataset-name random \
|
||||
--num-prompts $(E2E_NUM_PROMPTS) \
|
||||
--random-input-len $(E2E_INPUT_LEN) \
|
||||
--random-output-len $(E2E_OUTPUT_LEN) \
|
||||
--request-rate $(E2E_REQUEST_RATE) \
|
||||
--max-concurrency $(E2E_MAX_CONCURRENCY) \
|
||||
--warmup-requests 5 \
|
||||
--disable-tqdm || (echo "✗ E2E test failed"; exit 1); \
|
||||
fi
|
||||
@echo ""
|
||||
@echo "✓ E2E test completed"
|
||||
|
||||
# Run the server (development)
|
||||
run: build-dev
|
||||
@echo "Running server..."
|
||||
@export $(LD_LIBRARY_PATH_VAR)="$(LIB_DIR):$$$(LD_LIBRARY_PATH_VAR)" && \
|
||||
$(BINARY)
|
||||
|
||||
# Run streaming example
|
||||
stream: check-rust-lib
|
||||
@echo "Running streaming example..."
|
||||
@cd $(BINDINGS_DIR)/examples/streaming && \
|
||||
export $(LD_LIBRARY_PATH_VAR)="$(LIB_DIR):$$$(LD_LIBRARY_PATH_VAR)" && \
|
||||
bash run.sh
|
||||
|
||||
# Clean build artifacts
|
||||
clean:
|
||||
@echo "Cleaning build artifacts..."
|
||||
@rm -rf $(BUILD_DIR)
|
||||
@echo "✓ Clean complete"
|
||||
305
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/README.md
vendored
Normal file
305
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/README.md
vendored
Normal file
@@ -0,0 +1,305 @@
|
||||
# Go SGLang Router - OpenAI Compatible API Server
|
||||
|
||||
Go SGLang Router is a high-performance OpenAI-compatible API server that communicates with the SGLang backend via gRPC and performs efficient preprocessing and postprocessing through Rust FFI.
|
||||
|
||||
## Features
|
||||
|
||||
- ✅ **OpenAI API Compatible**: Fully compatible with OpenAI Chat Completions API
|
||||
- ✅ **High Performance**: Low latency and high throughput using gRPC and Rust FFI
|
||||
- ✅ **Streaming Support**: Server-Sent Events (SSE) streaming responses
|
||||
- ✅ **Thread-Safe**: Pre-created tokenizer handle, lock-free concurrency
|
||||
- ✅ **Graceful Shutdown**: Context cancellation mechanism to avoid resource leaks and panics
|
||||
- ✅ **Configurable**: Supports configuring channel buffer sizes and timeout durations
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
**Important Note**: gRPC mode **still calls FFI**, which is used for:
|
||||
- **Preprocessing**: chat_template and tokenization (request phase)
|
||||
- **Postprocessing**: token decoding and tool parsing (response phase)
|
||||
|
||||
gRPC is only used for communication with the SGLang backend, while input/output processing completely relies on Rust FFI.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ HTTP Client │
|
||||
│ (OpenAI API Format) │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ FastHTTP Server │
|
||||
│ handlers/chat.go:HandleChatCompletion │
|
||||
│ - Parse request JSON │
|
||||
│ - SetBodyStreamWriter (SSE) │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ SGLang Client (client.go) │
|
||||
│ CreateChatCompletionStream(ctx, req) │
|
||||
│ - Wraps gRPC client │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ gRPC Client (internal/grpc/client_grpc.go) │
|
||||
│ CreateChatCompletionStream(ctx, reqJSON) │
|
||||
│ │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 1: FFI Preprocess (Rust FFI) │ │
|
||||
│ │ - ffi.PreprocessChatRequestWithTokenizer() │ │
|
||||
│ │ - chat_template application │ │
|
||||
│ │ - tokenization │ │
|
||||
│ │ - tool constraints generation │ │
|
||||
│ │ Returns: PromptText, TokenIDs, ToolConstraintsJSON, │ │
|
||||
│ │ PromptTokens │ │
|
||||
│ └────────────────────┬─────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 2: Build gRPC Request │ │
|
||||
│ │ - Parse request JSON (model, temperature, etc.) │ │
|
||||
│ │ - Create proto.GenerateRequest │ │
|
||||
│ │ - Set TokenizedInput (PromptText, TokenIDs) │ │
|
||||
│ │ - Set SamplingParams (temperature, top_p, top_k, etc.) │ │
|
||||
│ │ - Set Constraints (from ToolConstraintsJSON) │ │
|
||||
│ └────────────────────┬─────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 3: Create gRPC Stream │ │
|
||||
│ │ - client.Generate(generateReq) → gRPC stream │ │
|
||||
│ │ - Connects to SGLang Backend (Rust) │ │
|
||||
│ └────────────────────┬─────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 4: Create Converter & BatchPostprocessor │ │
|
||||
│ │ - ffi.CreateGrpcResponseConverterWithTokenizer() │ │
|
||||
│ │ - Uses preprocessed.PromptTokens for initial count │ │
|
||||
│ │ - ffi.NewBatchPostprocessor(batchSize=1, immediate) │ │
|
||||
│ └────────────────────┬─────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ Step 5: Start readLoop (Background Goroutine) │ │
|
||||
│ │ - go grpcStream.readLoop() │ │
|
||||
│ │ - Returns GrpcChatCompletionStream immediately │ │
|
||||
│ └────────────────────┬─────────────────────────────────────┘ │
|
||||
└───────────────────────┼────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ GrpcChatCompletionStream.readLoop() │
|
||||
│ (Background Goroutine) │
|
||||
│ │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ Recv() Goroutine (Dedicated) │ │
|
||||
│ │ - Continuously calls stream.Recv() │ │
|
||||
│ │ - Sends results to recvChan (buffered, 2000) │ │
|
||||
│ │ - Exits on ctx.Done() or error │ │
|
||||
│ │ - Calls stream.CloseSend() on ctx.Done() │ │
|
||||
│ └────────────────────┬─────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ Main Loop │ │
|
||||
│ │ - Reads from recvChan │ │
|
||||
│ │ - For each proto.GenerateResponse: │ │
|
||||
│ │ → go processAndSendResponse() (async) │ │
|
||||
│ │ - protoToJSON() converts proto to JSON string │ │
|
||||
│ │ - batchPostprocessor.AddChunk(protoJSON) │ │
|
||||
│ │ → FFI postprocessing (token decoding, tool parsing)│ │
|
||||
│ │ → Returns OpenAI-format JSON strings │ │
|
||||
│ │ - Sends JSON to resultJSONChan (buffered, 10000) │ │
|
||||
│ │ - All operations check ctx.Done() for cancellation │ │
|
||||
│ │ - On EOF: flush batch, send remaining results, return │ │
|
||||
│ │ - On error: send to errChan (buffered, 100) │ │
|
||||
│ │ - defer: cancel ctx, wait goroutines, close channels │ │
|
||||
│ └────────────────────┬─────────────────────────────────────┘ │
|
||||
└───────────────────────┼────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ resultJSONChan (Buffered Channel, 10000) │
|
||||
│ - Contains OpenAI-format JSON strings │
|
||||
│ - Ready for consumption │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ ChatCompletionStream.RecvJSON() │
|
||||
│ (client.go:410) │
|
||||
│ - Direct wrapper: return grpcStream.RecvJSON() │
|
||||
│ - No intermediate processing │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ FastHTTP SetBodyStreamWriter │
|
||||
│ (handlers/chat.go:159) │
|
||||
│ - Loop: stream.RecvJSON() → format SSE → flush │
|
||||
│ - Format: "data: {json}\n\n" │
|
||||
│ - Final: "data: [DONE]\n\n" │
|
||||
│ - Immediate flush after each chunk │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ HTTP Client │
|
||||
│ (SSE Stream) │
|
||||
│ Receives: data: {...}\n\n │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Start Server
|
||||
|
||||
```bash
|
||||
./run.sh
|
||||
```
|
||||
|
||||
The server will start on port `:8080`.
|
||||
|
||||
### Usage Example
|
||||
|
||||
```bash
|
||||
curl http://localhost:8080/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "/path/to/model",
|
||||
"messages": [{"role": "user", "content": "Hello!"}],
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
## Key Design
|
||||
|
||||
### 1. Thread-Safe Tokenizer
|
||||
- Pre-create `TokenizerHandle` at startup
|
||||
- Rust side uses `Arc<dyn TokenizerTrait>`, thread-safe
|
||||
- Lock-free concurrency, eliminating lock contention
|
||||
|
||||
### 2. Context Cancellation Mechanism (Graceful Shutdown)
|
||||
- Use `context.Context` cancellation mechanism
|
||||
- In `readLoop`'s `defer`: cancel context first, then wait for all goroutines to complete, finally close channels
|
||||
- `processAndSendResponse` checks `ctx.Done()` at function start, all `select` statements include `case <-s.ctx.Done()`
|
||||
- Avoids "send on closed channel" panic
|
||||
|
||||
### 3. Cancellable Recv()
|
||||
- Use dedicated goroutine to execute `Recv()`
|
||||
- Pass results through `recvChan`
|
||||
- Call `CloseSend()` when context is cancelled to make `Recv()` return error
|
||||
|
||||
### 4. Simplified Channel Design
|
||||
- `resultJSONChan`: Main data channel (gRPC layer)
|
||||
- `errChan`: Error channel (gRPC layer)
|
||||
- `recvChan`: Internal communication channel (gRPC layer)
|
||||
- Removed redundant channels and duplicate reads
|
||||
|
||||
## Configuration
|
||||
|
||||
### Channel Buffer Sizes
|
||||
|
||||
```go
|
||||
type ChannelBufferSizes struct {
|
||||
ResultJSONChan int // Default: 10000
|
||||
ErrChan int // Default: 100
|
||||
RecvChan int // Default: 2000
|
||||
}
|
||||
```
|
||||
|
||||
### Timeout Configuration
|
||||
|
||||
```go
|
||||
type Timeouts struct {
|
||||
KeepaliveTime time.Duration // Default: 300s
|
||||
KeepaliveTimeout time.Duration // Default: 20s
|
||||
CloseTimeout time.Duration // Default: 5s
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Optimizations
|
||||
|
||||
1. **Pre-create Tokenizer**: Created at startup to avoid first request latency
|
||||
2. **Lock-Free Concurrency**: Tokenizer is thread-safe, no locks needed
|
||||
3. **Lazy Parsing**: JSON parsing deferred until needed
|
||||
4. **Direct JSON Passing**: `RecvJSON()` avoids parse/serialize overhead
|
||||
5. **Immediate Batching**: batchSize=1, no delay
|
||||
6. **Async Processing**: `readLoop` processes in background, doesn't block request handling
|
||||
7. **Configurable Buffers**: Adjust channel sizes based on concurrency needs
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
sgl-model-gateway/bindings/golang/
|
||||
├── client.go # High-level client API
|
||||
├── internal/
|
||||
│ ├── grpc/
|
||||
│ │ └── client_grpc.go # gRPC client implementation
|
||||
│ ├── ffi/ # FFI bindings (Rust)
|
||||
│ └── proto/ # Protobuf definitions
|
||||
└── examples/
|
||||
└── oai_server/
|
||||
├── handlers/
|
||||
│ └── chat.go # HTTP request handling
|
||||
├── models/
|
||||
│ └── chat.go # Request/response models
|
||||
└── service/
|
||||
└── sglang_service.go # Service layer
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Context Cancellation Mechanism
|
||||
1. **Client disconnects** → `SetBodyStreamWriter` detects flush error
|
||||
2. **Cancel streamCtx** → `readLoop` detects `ctx.Done()`
|
||||
3. **Call stream.CloseSend()** → `Recv()` goroutine returns error
|
||||
4. **readLoop defer executes**:
|
||||
- Set `closed` flag
|
||||
- Cancel context (if not already cancelled)
|
||||
- Wait for all `processAndSendResponse` goroutines to complete (`processWg.Wait()`)
|
||||
- Close all channels (`resultJSONChan`, `errChan`, `readLoopDone`)
|
||||
5. **Clean up resources and exit**
|
||||
|
||||
### Channel Blocking and Race Condition Prevention
|
||||
- **Context cancellation mechanism**: All channel sends use `select` statements with `case <-s.ctx.Done()`
|
||||
- **Graceful exit**: When context is cancelled, all blocking send operations can return immediately
|
||||
- **WaitGroup synchronization**: `readLoop`'s `defer` uses `processWg.Wait()` to ensure all goroutines complete before closing channels
|
||||
- **Avoid panic**: Through context cancellation and WaitGroup synchronization, avoids "send on closed channel" panic
|
||||
|
||||
## Key Functions
|
||||
|
||||
### CreateChatCompletionStream
|
||||
**Location**: `internal/grpc/client_grpc.go:108`
|
||||
- Preprocess request (FFI)
|
||||
- Build gRPC request
|
||||
- Create converter and batch processor
|
||||
- Start `readLoop`
|
||||
|
||||
### readLoop
|
||||
**Location**: `internal/grpc/client_grpc.go:290`
|
||||
- Start Recv() goroutine (continuously calls `stream.Recv()`)
|
||||
- Process proto responses
|
||||
- Asynchronously call `processAndSendResponse` (tracked with `processWg`)
|
||||
- **Graceful shutdown in defer**:
|
||||
- Set `closed` flag
|
||||
- Cancel context (if not already cancelled)
|
||||
- Wait for all `processAndSendResponse` goroutines to complete (`processWg.Wait()`)
|
||||
- Close all channels (`resultJSONChan`, `errChan`, `readLoopDone`)
|
||||
|
||||
### processAndSendResponse
|
||||
**Location**: `internal/grpc/client_grpc.go:379`
|
||||
- Check `ctx.Done()` at function start, return immediately if cancelled
|
||||
- Convert proto to JSON
|
||||
- Call FFI batch processor
|
||||
- All `select` statements include `case <-s.ctx.Done()` for graceful shutdown handling
|
||||
- Send JSON to channel
|
||||
|
||||
### RecvJSON
|
||||
**Location**:
|
||||
- `internal/grpc/client_grpc.go:412`: gRPC layer implementation
|
||||
- `client.go:410`: Client wrapper layer
|
||||
- Read from `resultJSONChan`
|
||||
- Directly return JSON string, no parsing needed
|
||||
55
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/config/config.go
vendored
Normal file
55
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/config/config.go
vendored
Normal file
@@ -0,0 +1,55 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Config holds the application configuration
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
TokenizerPath string
|
||||
Port string
|
||||
LogDir string
|
||||
LogLevel string
|
||||
}
|
||||
|
||||
// Load loads configuration from environment variables with defaults
|
||||
func Load() *Config {
|
||||
// Get tokenizer path from environment or use default
|
||||
tokenizerPath := os.Getenv("SGL_TOKENIZER_PATH")
|
||||
if tokenizerPath == "" {
|
||||
tokenizerPath = "../tokenizer"
|
||||
}
|
||||
|
||||
// Get endpoint from environment or use default
|
||||
endpoint := os.Getenv("SGL_GRPC_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
endpoint = "grpc://localhost:20000"
|
||||
}
|
||||
|
||||
// Get port from environment or use default
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
|
||||
// Get log directory from environment or use default
|
||||
logDir := os.Getenv("LOG_DIR")
|
||||
if logDir == "" {
|
||||
logDir = "./logs"
|
||||
}
|
||||
|
||||
// Get log level from environment or use default
|
||||
logLevel := os.Getenv("LOG_LEVEL")
|
||||
if logLevel == "" {
|
||||
logLevel = "info"
|
||||
}
|
||||
|
||||
return &Config{
|
||||
Endpoint: endpoint,
|
||||
TokenizerPath: tokenizerPath,
|
||||
Port: port,
|
||||
LogDir: logDir,
|
||||
LogLevel: logLevel,
|
||||
}
|
||||
}
|
||||
121
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/docs/benchmark_result.md
vendored
Normal file
121
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/docs/benchmark_result.md
vendored
Normal file
@@ -0,0 +1,121 @@
|
||||
/tmp/ShareGPT_V3_unfiltered_cleaned_split.json: 100%|████████████████████| 642M/642M [10:02<00:00, 1.12MB/s]
|
||||
#Input tokens: 50561
|
||||
#Output tokens: 25883
|
||||
Starting warmup with 5 sequences...
|
||||
Warmup completed with 5 sequences. Starting main benchmark run...
|
||||
|
||||
============ Serving Benchmark Result ============
|
||||
Backend: sglang-oai-chat
|
||||
Traffic request rate: 20.0
|
||||
Max request concurrency: 20
|
||||
Successful requests: 100
|
||||
Benchmark duration (s): 107.24
|
||||
Total input tokens: 50561
|
||||
Total input text tokens: 50561
|
||||
Total input vision tokens: 0
|
||||
Total generated tokens: 25883
|
||||
Total generated tokens (retokenized): 129591
|
||||
Request throughput (req/s): 0.93
|
||||
Input token throughput (tok/s): 471.48
|
||||
Output token throughput (tok/s): 241.36
|
||||
Total token throughput (tok/s): 712.84
|
||||
Concurrency: 16.42
|
||||
----------------End-to-End Latency----------------
|
||||
Mean E2E Latency (ms): 17609.46
|
||||
Median E2E Latency (ms): 12343.82
|
||||
---------------Time to First Token----------------
|
||||
Mean TTFT (ms): 190.71
|
||||
Median TTFT (ms): 164.86
|
||||
P99 TTFT (ms): 397.72
|
||||
-----Time per Output Token (excl. 1st token)------
|
||||
Mean TPOT (ms): 162.55
|
||||
Median TPOT (ms): 63.51
|
||||
P99 TPOT (ms): 1337.20
|
||||
---------------Inter-Token Latency----------------
|
||||
Mean ITL (ms): 25.85
|
||||
Median ITL (ms): 24.26
|
||||
P95 ITL (ms): 48.26
|
||||
P99 ITL (ms): 119.04
|
||||
Max ITL (ms): 194.58
|
||||
==================================================
|
||||
|
||||
✓ E2E test completed
|
||||
|
||||
|
||||
## Rust
|
||||
============ Serving Benchmark Result ============
|
||||
Backend: sglang-oai-chat
|
||||
Traffic request rate: 20.0
|
||||
Max request concurrency: 20
|
||||
Successful requests: 100
|
||||
Benchmark duration (s): 37.71
|
||||
Total input tokens: 50561
|
||||
Total input text tokens: 50561
|
||||
Total input vision tokens: 0
|
||||
Total generated tokens: 25883
|
||||
Total generated tokens (retokenized): 25599
|
||||
Request throughput (req/s): 2.65
|
||||
Input token throughput (tok/s): 1340.75
|
||||
Output token throughput (tok/s): 686.35
|
||||
Total token throughput (tok/s): 2027.10
|
||||
Concurrency: 18.58
|
||||
----------------End-to-End Latency----------------
|
||||
Mean E2E Latency (ms): 7008.05
|
||||
Median E2E Latency (ms): 7061.24
|
||||
---------------Time to First Token----------------
|
||||
Mean TTFT (ms): 156.09
|
||||
Median TTFT (ms): 133.81
|
||||
P99 TTFT (ms): 318.53
|
||||
-----Time per Output Token (excl. 1st token)------
|
||||
Mean TPOT (ms): 26.59
|
||||
Median TPOT (ms): 26.75
|
||||
P99 TPOT (ms): 29.18
|
||||
---------------Inter-Token Latency----------------
|
||||
Mean ITL (ms): 26.71
|
||||
Median ITL (ms): 23.61
|
||||
P95 ITL (ms): 66.11
|
||||
P99 ITL (ms): 115.30
|
||||
Max ITL (ms): 201.08
|
||||
==================================================
|
||||
|
||||
|
||||
## golang
|
||||
#Input tokens: 50561
|
||||
#Output tokens: 25883
|
||||
Starting warmup with 5 sequences...
|
||||
Warmup completed with 5 sequences. Starting main benchmark run...
|
||||
|
||||
============ Serving Benchmark Result ============
|
||||
Backend: sglang-oai-chat
|
||||
Traffic request rate: 20.0
|
||||
Max request concurrency: 20
|
||||
Successful requests: 100
|
||||
Benchmark duration (s): 34.22
|
||||
Total input tokens: 50561
|
||||
Total input text tokens: 50561
|
||||
Total input vision tokens: 0
|
||||
Total generated tokens: 22970
|
||||
Total generated tokens (retokenized): 31740
|
||||
Request throughput (req/s): 2.92
|
||||
Input token throughput (tok/s): 1477.70
|
||||
Output token throughput (tok/s): 671.32
|
||||
Total token throughput (tok/s): 2149.03
|
||||
Concurrency: 18.42
|
||||
----------------End-to-End Latency----------------
|
||||
Mean E2E Latency (ms): 6303.33
|
||||
Median E2E Latency (ms): 6294.46
|
||||
---------------Time to First Token----------------
|
||||
Mean TTFT (ms): 157.10
|
||||
Median TTFT (ms): 149.16
|
||||
P99 TTFT (ms): 251.98
|
||||
-----Time per Output Token (excl. 1st token)------
|
||||
Mean TPOT (ms): 26.49
|
||||
Median TPOT (ms): 27.15
|
||||
P99 TPOT (ms): 28.73
|
||||
---------------Inter-Token Latency----------------
|
||||
Mean ITL (ms): 26.97
|
||||
Median ITL (ms): 24.61
|
||||
P95 ITL (ms): 52.39
|
||||
P99 ITL (ms): 86.52
|
||||
Max ITL (ms): 194.55
|
||||
==================================================
|
||||
60
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/go.sum
vendored
Normal file
60
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/go.sum
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
||||
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
|
||||
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
|
||||
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo=
|
||||
golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
|
||||
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
||||
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
|
||||
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
|
||||
google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
556
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/handlers/chat.go
vendored
Normal file
556
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/handlers/chat.go
vendored
Normal file
@@ -0,0 +1,556 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sglang "github.com/sglang/sglang-go-grpc-sdk"
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"oai_server/models"
|
||||
"oai_server/service"
|
||||
"oai_server/utils"
|
||||
)
|
||||
|
||||
// ChatHandler handles chat completion requests
|
||||
type ChatHandler struct {
|
||||
logger *zap.Logger
|
||||
service *service.SGLangService
|
||||
}
|
||||
|
||||
// NewChatHandler creates a new chat handler
|
||||
func NewChatHandler(logger *zap.Logger, svc *service.SGLangService) *ChatHandler {
|
||||
return &ChatHandler{
|
||||
logger: logger,
|
||||
service: svc,
|
||||
}
|
||||
}
|
||||
|
||||
// recvResult holds the result of a RecvJSON() call
|
||||
type recvResult struct {
|
||||
chunkJSON string
|
||||
err error
|
||||
}
|
||||
|
||||
// HandleChatCompletion handles POST /v1/chat/completions
|
||||
func (h *ChatHandler) HandleChatCompletion(ctx *fasthttp.RequestCtx) {
|
||||
var req models.ChatRequest
|
||||
if err := json.Unmarshal(ctx.PostBody(), &req); err != nil {
|
||||
h.logger.Warn("Invalid chat completion request", zap.Error(err))
|
||||
utils.RespondError(ctx, 400, fmt.Sprintf("Invalid request: %v", err), "invalid_request_error")
|
||||
return
|
||||
}
|
||||
|
||||
path := string(ctx.Path())
|
||||
|
||||
defer func() {
|
||||
statusCode := ctx.Response.StatusCode()
|
||||
if statusCode == 0 {
|
||||
statusCode = 200
|
||||
}
|
||||
h.logHTTPResponse(statusCode, path)
|
||||
}()
|
||||
|
||||
// Convert to SGLang format
|
||||
messages := make([]sglang.ChatMessage, len(req.Messages))
|
||||
for i, msg := range req.Messages {
|
||||
role, roleOk := msg["role"]
|
||||
content, contentOk := msg["content"]
|
||||
|
||||
// Validate role
|
||||
if !roleOk || role == "" {
|
||||
h.logger.Warn("Missing or empty role in message", zap.Int("message_index", i))
|
||||
utils.RespondError(ctx, 400, "Message role is required and cannot be empty", "invalid_request_error")
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure content is always a string (not null)
|
||||
// Chat template requires content field to be present, even if empty
|
||||
// If content is missing or null, use empty string
|
||||
contentStr := ""
|
||||
if contentOk && content != "" {
|
||||
contentStr = content
|
||||
}
|
||||
|
||||
messages[i] = sglang.ChatMessage{
|
||||
Role: role,
|
||||
Content: contentStr,
|
||||
}
|
||||
}
|
||||
|
||||
sglReq := sglang.ChatCompletionRequest{
|
||||
Model: req.Model,
|
||||
Messages: messages,
|
||||
Stream: req.Stream,
|
||||
}
|
||||
|
||||
if req.Temperature != nil {
|
||||
temp := float32(*req.Temperature)
|
||||
sglReq.Temperature = &temp
|
||||
}
|
||||
if req.TopP != nil {
|
||||
topP := float32(*req.TopP)
|
||||
sglReq.TopP = &topP
|
||||
}
|
||||
if req.MaxCompletionTokens != nil {
|
||||
sglReq.MaxCompletionTokens = req.MaxCompletionTokens
|
||||
} else if req.MaxTokens != nil {
|
||||
sglReq.MaxCompletionTokens = req.MaxTokens
|
||||
}
|
||||
|
||||
requestCtx := context.Background()
|
||||
|
||||
if req.Stream {
|
||||
h.handleStreamingCompletion(ctx, requestCtx, sglReq)
|
||||
} else {
|
||||
h.handleNonStreamingCompletion(ctx, requestCtx, sglReq)
|
||||
}
|
||||
}
|
||||
|
||||
// isBrokenPipeError checks if the error is a broken pipe error (client disconnected)
|
||||
func isBrokenPipeError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
errStr := err.Error()
|
||||
return strings.Contains(errStr, "broken pipe") ||
|
||||
strings.Contains(errStr, "connection reset by peer") ||
|
||||
strings.Contains(errStr, "connection closed") ||
|
||||
strings.Contains(errStr, "write: connection closed")
|
||||
}
|
||||
|
||||
// logHTTPResponse logs HTTP response with colored output
|
||||
func (h *ChatHandler) logHTTPResponse(statusCode int, path string) {
|
||||
var statusText string
|
||||
var colorCode string
|
||||
|
||||
switch {
|
||||
case statusCode >= 200 && statusCode < 300:
|
||||
colorCode = "\033[32m" // Green
|
||||
statusText = "OK"
|
||||
case statusCode >= 300 && statusCode < 400:
|
||||
colorCode = "\033[33m" // Yellow
|
||||
statusText = "Redirect"
|
||||
case statusCode >= 400 && statusCode < 500:
|
||||
colorCode = "\033[33m" // Yellow
|
||||
statusText = "Client Error"
|
||||
case statusCode >= 500:
|
||||
colorCode = "\033[31m" // Red
|
||||
statusText = "Server Error"
|
||||
default:
|
||||
colorCode = "\033[37m" // White
|
||||
statusText = "Unknown"
|
||||
}
|
||||
|
||||
resetCode := "\033[0m"
|
||||
msg := fmt.Sprintf("%s[%d %s]%s %s", colorCode, statusCode, statusText, resetCode, path)
|
||||
h.logger.Info(msg)
|
||||
}
|
||||
|
||||
func (h *ChatHandler) handleStreamingCompletion(ctx *fasthttp.RequestCtx, requestCtx context.Context, req sglang.ChatCompletionRequest) {
|
||||
|
||||
ctx.SetContentType("text/event-stream")
|
||||
ctx.Response.Header.Set("Cache-Control", "no-cache")
|
||||
ctx.Response.Header.Set("Connection", "keep-alive")
|
||||
ctx.Response.Header.Set("X-Accel-Buffering", "no")
|
||||
ctx.SetStatusCode(200)
|
||||
|
||||
var clientDisconnected bool
|
||||
// Flush timeout: prevent deadlock if client is slow or disconnected
|
||||
// This timeout should be longer than typical network latency but shorter than client timeout
|
||||
const flushTimeout = 5 * time.Second
|
||||
|
||||
ctx.SetBodyStreamWriter(func(w *bufio.Writer) {
|
||||
streamCtx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
stream, err := h.service.Client().CreateChatCompletionStream(streamCtx, req)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to create chat completion stream",
|
||||
zap.Error(err),
|
||||
zap.String("model", req.Model),
|
||||
)
|
||||
// Use sendSSEError to send error in consistent format
|
||||
errInfo, sendErr := h.sendSSEError(w, err)
|
||||
if sendErr != nil {
|
||||
h.logger.Warn("Failed to send SSE error", zap.Error(sendErr))
|
||||
} else if errInfo.IsTimeout {
|
||||
h.logger.Error("Stream creation timeout", zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := stream.Close(); closeErr != nil {
|
||||
h.logger.Warn("Failed to close stream", zap.Error(closeErr))
|
||||
}
|
||||
}()
|
||||
|
||||
// Use a single dedicated goroutine to continuously call RecvJSON() and send results via channel
|
||||
recvChan := make(chan recvResult, 20)
|
||||
recvGoroutineDone := make(chan struct{})
|
||||
go func() {
|
||||
defer func() {
|
||||
close(recvChan)
|
||||
close(recvGoroutineDone)
|
||||
}()
|
||||
for {
|
||||
// Check context before calling RecvJSON() to avoid blocking if context is cancelled
|
||||
select {
|
||||
case <-streamCtx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// Call RecvJSON() - this may block, but stream.Close() will unblock it
|
||||
// when context is cancelled (called from main loop)
|
||||
chunkJSON, err := stream.RecvJSON()
|
||||
|
||||
// Check context again after RecvJSON() returns
|
||||
select {
|
||||
case <-streamCtx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// Send to channel (may block if channel is full)
|
||||
// If channel is full, this will block until main loop reads from it
|
||||
// This is acceptable because main loop should be actively reading
|
||||
select {
|
||||
case recvChan <- recvResult{chunkJSON: chunkJSON, err: err}:
|
||||
if err != nil {
|
||||
// EOF or other error, stop the goroutine
|
||||
return
|
||||
}
|
||||
case <-streamCtx.Done():
|
||||
// Context cancelled while sending, stop the goroutine
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
if clientDisconnected {
|
||||
cancel()
|
||||
// Close stream immediately to unblock RecvJSON() calls
|
||||
stream.Close()
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case <-streamCtx.Done():
|
||||
// Close stream to ensure RecvJSON() goroutine can exit
|
||||
stream.Close()
|
||||
return
|
||||
case result, ok := <-recvChan:
|
||||
if !ok {
|
||||
// Channel closed, stream ended
|
||||
return
|
||||
}
|
||||
if result.err == io.EOF {
|
||||
if !clientDisconnected {
|
||||
w.WriteString("data: [DONE]\n\n")
|
||||
// Flush with timeout to prevent deadlock
|
||||
flushDone := make(chan error, 1)
|
||||
go func() {
|
||||
flushDone <- w.Flush()
|
||||
}()
|
||||
flushCtx, flushCancel := context.WithTimeout(streamCtx, flushTimeout)
|
||||
defer flushCancel()
|
||||
select {
|
||||
case flushErr := <-flushDone:
|
||||
if flushErr != nil && !isBrokenPipeError(flushErr) {
|
||||
h.logger.Warn("Final flush error", zap.Error(flushErr))
|
||||
}
|
||||
case <-flushCtx.Done():
|
||||
if flushCtx.Err() == context.DeadlineExceeded {
|
||||
h.logger.Warn("Final flush timeout", zap.Duration("timeout", flushTimeout))
|
||||
}
|
||||
case <-streamCtx.Done():
|
||||
// Context cancelled, skip flush
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.err != nil {
|
||||
if result.err == context.Canceled || result.err == context.DeadlineExceeded {
|
||||
return
|
||||
}
|
||||
// Send error to client before closing
|
||||
errInfo, sendErr := h.sendSSEError(w, result.err)
|
||||
if sendErr != nil {
|
||||
h.logger.Warn("Failed to send SSE error", zap.Error(sendErr))
|
||||
}
|
||||
if errInfo.IsTimeout {
|
||||
h.logger.Error("Stream timeout error", zap.Error(result.err))
|
||||
} else {
|
||||
h.logger.Error("Stream error", zap.Error(result.err))
|
||||
}
|
||||
return
|
||||
}
|
||||
if result.chunkJSON == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
w.WriteString("data: ")
|
||||
w.WriteString(result.chunkJSON)
|
||||
w.WriteString("\n\n")
|
||||
|
||||
// Flush with timeout to prevent deadlock:
|
||||
// If Flush blocks indefinitely (slow client), RecvJSON goroutine may fill recvChan
|
||||
// and then block trying to send, causing deadlock
|
||||
// Note: bufio.Writer.Flush() doesn't have a timeout parameter, so we use
|
||||
// a goroutine + select pattern to implement timeout behavior
|
||||
flushDone := make(chan error, 1)
|
||||
go func() {
|
||||
flushDone <- w.Flush()
|
||||
}()
|
||||
|
||||
flushCtx, flushCancel := context.WithTimeout(streamCtx, flushTimeout)
|
||||
defer flushCancel()
|
||||
|
||||
select {
|
||||
case err := <-flushDone:
|
||||
if err != nil {
|
||||
if isBrokenPipeError(err) {
|
||||
clientDisconnected = true
|
||||
cancel()
|
||||
// Close stream immediately to unblock RecvJSON() calls
|
||||
stream.Close()
|
||||
return
|
||||
}
|
||||
h.logger.Warn("Flush error", zap.Error(err))
|
||||
}
|
||||
case <-flushCtx.Done():
|
||||
// Flush timeout: client may be slow or disconnected
|
||||
// Continue processing to avoid deadlock, but mark as disconnected
|
||||
if flushCtx.Err() == context.DeadlineExceeded {
|
||||
h.logger.Warn("Flush timeout, client may be slow or disconnected", zap.Duration("timeout", flushTimeout))
|
||||
}
|
||||
clientDisconnected = true
|
||||
cancel()
|
||||
stream.Close()
|
||||
return
|
||||
case <-streamCtx.Done():
|
||||
// Context cancelled, stop flushing
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (h *ChatHandler) handleNonStreamingCompletion(ctx *fasthttp.RequestCtx, requestCtx context.Context, req sglang.ChatCompletionRequest) {
|
||||
resp, err := h.service.Client().CreateChatCompletion(requestCtx, req)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to create chat completion",
|
||||
zap.Error(err),
|
||||
zap.String("model", req.Model),
|
||||
)
|
||||
utils.RespondError(ctx, 500, fmt.Sprintf("Failed to create completion: %v", err), "server_error")
|
||||
return
|
||||
}
|
||||
|
||||
// Convert to OpenAI format
|
||||
response := utils.BuildResponseBase(resp.ID, resp.Created, resp.Model)
|
||||
response["object"] = "chat.completion"
|
||||
|
||||
choices := make([]map[string]interface{}, len(resp.Choices))
|
||||
for i, choice := range resp.Choices {
|
||||
choiceMap := map[string]interface{}{
|
||||
"index": choice.Index,
|
||||
"message": map[string]interface{}{
|
||||
"role": choice.Message.Role,
|
||||
"content": choice.Message.Content,
|
||||
},
|
||||
"finish_reason": choice.FinishReason,
|
||||
}
|
||||
if len(choice.Message.ToolCalls) > 0 {
|
||||
toolCalls := make([]map[string]interface{}, len(choice.Message.ToolCalls))
|
||||
for j, tc := range choice.Message.ToolCalls {
|
||||
toolCalls[j] = map[string]interface{}{
|
||||
"id": tc.ID,
|
||||
"type": tc.Type,
|
||||
"function": map[string]interface{}{"name": tc.Function.Name, "arguments": tc.Function.Arguments},
|
||||
}
|
||||
}
|
||||
choiceMap["message"].(map[string]interface{})["tool_calls"] = toolCalls
|
||||
}
|
||||
choices[i] = choiceMap
|
||||
}
|
||||
response["choices"] = choices
|
||||
|
||||
// Usage is always present (not a pointer)
|
||||
response["usage"] = map[string]interface{}{
|
||||
"prompt_tokens": resp.Usage.PromptTokens,
|
||||
"completion_tokens": resp.Usage.CompletionTokens,
|
||||
"total_tokens": resp.Usage.TotalTokens,
|
||||
}
|
||||
|
||||
ctx.SetStatusCode(200)
|
||||
ctx.SetContentType("application/json")
|
||||
jsonData, _ := json.Marshal(response)
|
||||
ctx.Write(jsonData)
|
||||
}
|
||||
|
||||
// StreamErrorInfo holds parsed error information
|
||||
type StreamErrorInfo struct {
|
||||
Message string
|
||||
Type string
|
||||
Code int
|
||||
IsTimeout bool
|
||||
}
|
||||
|
||||
// parseStreamError parses error type and code
|
||||
func parseStreamError(err error) StreamErrorInfo {
|
||||
if err == nil {
|
||||
return StreamErrorInfo{}
|
||||
}
|
||||
|
||||
errorMsg := err.Error()
|
||||
// Check timeout error by message prefix
|
||||
isTimeout := strings.HasPrefix(errorMsg, "stream.Recv() timeout") || strings.Contains(errorMsg, "timeout after")
|
||||
|
||||
errorType := "server_error"
|
||||
errorCode := 500
|
||||
if isTimeout {
|
||||
errorType = "timeout_error"
|
||||
errorCode = 504
|
||||
}
|
||||
|
||||
return StreamErrorInfo{
|
||||
Message: errorMsg,
|
||||
Type: errorType,
|
||||
Code: errorCode,
|
||||
IsTimeout: isTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
// formatErrorJSON formats error as OpenAI JSON
|
||||
func formatErrorJSON(errInfo StreamErrorInfo) string {
|
||||
errorObj := map[string]interface{}{
|
||||
"error": map[string]interface{}{
|
||||
"message": errInfo.Message,
|
||||
"type": errInfo.Type,
|
||||
"code": errInfo.Code,
|
||||
},
|
||||
}
|
||||
jsonBytes, _ := json.Marshal(errorObj)
|
||||
return string(jsonBytes)
|
||||
}
|
||||
|
||||
// sendSSEError sends SSE error response. Callers should log errors.
|
||||
func (h *ChatHandler) sendSSEError(w *bufio.Writer, err error) (StreamErrorInfo, error) {
|
||||
errInfo := parseStreamError(err)
|
||||
errorJSON := formatErrorJSON(errInfo)
|
||||
|
||||
w.WriteString("data: ")
|
||||
w.WriteString(errorJSON)
|
||||
w.WriteString("\n\n")
|
||||
|
||||
if flushErr := w.Flush(); flushErr != nil && !isBrokenPipeError(flushErr) {
|
||||
h.logger.Warn("Failed to flush error response", zap.Error(flushErr))
|
||||
return errInfo, flushErr
|
||||
}
|
||||
|
||||
return errInfo, nil
|
||||
}
|
||||
|
||||
// HandleGenerate handles POST /generate (SGLang native API)
|
||||
func (h *ChatHandler) HandleGenerate(ctx *fasthttp.RequestCtx) {
|
||||
path := string(ctx.Path())
|
||||
|
||||
defer func() {
|
||||
statusCode := ctx.Response.StatusCode()
|
||||
if statusCode == 0 {
|
||||
statusCode = 200
|
||||
}
|
||||
h.logHTTPResponse(statusCode, path)
|
||||
}()
|
||||
|
||||
// Parse request body
|
||||
var req map[string]interface{}
|
||||
if err := json.Unmarshal(ctx.PostBody(), &req); err != nil {
|
||||
h.logger.Warn("Invalid generate request", zap.Error(err))
|
||||
utils.RespondError(ctx, 400, fmt.Sprintf("Invalid request: %v", err), "invalid_request_error")
|
||||
return
|
||||
}
|
||||
|
||||
// Extract text and sampling_params
|
||||
text, ok := req["text"].(string)
|
||||
if !ok || text == "" {
|
||||
utils.RespondError(ctx, 400, "Missing or invalid 'text' field", "invalid_request_error")
|
||||
return
|
||||
}
|
||||
|
||||
samplingParams, _ := req["sampling_params"].(map[string]interface{})
|
||||
if samplingParams == nil {
|
||||
samplingParams = make(map[string]interface{})
|
||||
}
|
||||
|
||||
// Convert to chat completion format for processing
|
||||
chatReq := sglang.ChatCompletionRequest{
|
||||
Model: "default",
|
||||
Messages: []sglang.ChatMessage{{Role: "user", Content: text}},
|
||||
Stream: false,
|
||||
}
|
||||
|
||||
// Copy sampling params
|
||||
if maxNewTokens, ok := samplingParams["max_new_tokens"].(float64); ok {
|
||||
tokens := int(maxNewTokens)
|
||||
chatReq.MaxCompletionTokens = &tokens
|
||||
}
|
||||
if temp, ok := samplingParams["temperature"].(float64); ok {
|
||||
temp32 := float32(temp)
|
||||
chatReq.Temperature = &temp32
|
||||
}
|
||||
if topP, ok := samplingParams["top_p"].(float64); ok {
|
||||
topP32 := float32(topP)
|
||||
chatReq.TopP = &topP32
|
||||
}
|
||||
if topK, ok := samplingParams["top_k"].(float64); ok {
|
||||
topKInt := int(topK)
|
||||
chatReq.TopK = &topKInt
|
||||
}
|
||||
|
||||
requestCtx := context.Background()
|
||||
|
||||
// Use non-streaming completion for /generate endpoint
|
||||
resp, err := h.service.Client().CreateChatCompletion(requestCtx, chatReq)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to create completion",
|
||||
zap.Error(err),
|
||||
)
|
||||
utils.RespondError(ctx, 500, fmt.Sprintf("Failed to create completion: %v", err), "server_error")
|
||||
return
|
||||
}
|
||||
|
||||
// Convert to SGLang /generate response format
|
||||
// meta_info must match SGLang's expected format with completion_tokens at top level
|
||||
finishReason := resp.Choices[0].FinishReason
|
||||
if finishReason == "" {
|
||||
finishReason = "stop"
|
||||
}
|
||||
|
||||
response := map[string]interface{}{
|
||||
"text": resp.Choices[0].Message.Content,
|
||||
"meta_info": map[string]interface{}{
|
||||
"id": resp.ID,
|
||||
"finish_reason": finishReason,
|
||||
"prompt_tokens": resp.Usage.PromptTokens,
|
||||
"completion_tokens": resp.Usage.CompletionTokens,
|
||||
"cached_tokens": 0, // Not available from chat completion API
|
||||
"weight_version": "", // Not available from chat completion API
|
||||
},
|
||||
}
|
||||
|
||||
ctx.SetStatusCode(200)
|
||||
ctx.SetContentType("application/json")
|
||||
jsonData, _ := json.Marshal(response)
|
||||
ctx.Write(jsonData)
|
||||
}
|
||||
33
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/handlers/health.go
vendored
Normal file
33
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/handlers/health.go
vendored
Normal file
@@ -0,0 +1,33 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// HealthHandler handles health check requests
|
||||
type HealthHandler struct {
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
// NewHealthHandler creates a new health handler
|
||||
func NewHealthHandler(logger *zap.Logger) *HealthHandler {
|
||||
return &HealthHandler{
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// Check handles GET /health
|
||||
func (h *HealthHandler) Check(ctx *fasthttp.RequestCtx) {
|
||||
ctx.SetStatusCode(200)
|
||||
ctx.SetContentType("application/json")
|
||||
|
||||
response := map[string]string{
|
||||
"status": "ok",
|
||||
}
|
||||
|
||||
jsonData, _ := json.Marshal(response)
|
||||
ctx.Write(jsonData)
|
||||
}
|
||||
67
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/handlers/models.go
vendored
Normal file
67
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/handlers/models.go
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// ModelsHandler handles model list requests
|
||||
type ModelsHandler struct {
|
||||
logger *zap.Logger
|
||||
tokenizerPath string
|
||||
}
|
||||
|
||||
// NewModelsHandler creates a new models handler
|
||||
func NewModelsHandler(logger *zap.Logger, tokenizerPath string) *ModelsHandler {
|
||||
return &ModelsHandler{
|
||||
logger: logger,
|
||||
tokenizerPath: tokenizerPath,
|
||||
}
|
||||
}
|
||||
|
||||
// List handles GET /v1/models
|
||||
func (h *ModelsHandler) List(ctx *fasthttp.RequestCtx) {
|
||||
// Return a default model for OpenAI compatibility
|
||||
ctx.SetStatusCode(200)
|
||||
ctx.SetContentType("application/json")
|
||||
|
||||
response := map[string]interface{}{
|
||||
"object": "list",
|
||||
"data": []map[string]interface{}{
|
||||
{
|
||||
"id": "default",
|
||||
"object": "model",
|
||||
"created": 1677610602,
|
||||
"owned_by": "sglang",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
jsonData, _ := json.Marshal(response)
|
||||
ctx.Write(jsonData)
|
||||
}
|
||||
|
||||
// GetModelInfo handles GET /get_model_info
|
||||
// Returns model information compatible with SGLang RuntimeEndpoint
|
||||
func (h *ModelsHandler) GetModelInfo(ctx *fasthttp.RequestCtx) {
|
||||
ctx.SetStatusCode(200)
|
||||
ctx.SetContentType("application/json")
|
||||
|
||||
// Return model info compatible with SGLang RuntimeEndpoint expectations
|
||||
response := map[string]interface{}{
|
||||
"model_path": h.tokenizerPath, // Use tokenizer path as model path
|
||||
"tokenizer_path": h.tokenizerPath,
|
||||
"is_generation": true,
|
||||
"preferred_sampling_params": "",
|
||||
"weight_version": "",
|
||||
"has_image_understanding": false,
|
||||
"has_audio_understanding": false,
|
||||
"model_type": "",
|
||||
"architectures": nil,
|
||||
}
|
||||
|
||||
jsonData, _ := json.Marshal(response)
|
||||
ctx.Write(jsonData)
|
||||
}
|
||||
67
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/logger/logger.go
vendored
Normal file
67
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/logger/logger.go
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
package logger
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"gopkg.in/natefinch/lumberjack.v2"
|
||||
)
|
||||
|
||||
// Init initializes the logger with file and console output
|
||||
func Init(logDir, logLevel string) (*zap.Logger, error) {
|
||||
// Ensure log directory exists
|
||||
if err := os.MkdirAll(logDir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse log level
|
||||
var level zapcore.Level
|
||||
if err := level.UnmarshalText([]byte(logLevel)); err != nil {
|
||||
level = zapcore.InfoLevel
|
||||
}
|
||||
|
||||
// Create log file path with date
|
||||
logFile := filepath.Join(logDir, "oai_server-"+time.Now().Format("2006-01-02")+".log")
|
||||
|
||||
// File writer with rotation
|
||||
fileWriter := zapcore.AddSync(&lumberjack.Logger{
|
||||
Filename: logFile,
|
||||
MaxSize: 100, // megabytes
|
||||
MaxBackups: 10,
|
||||
MaxAge: 30, // days
|
||||
Compress: true,
|
||||
})
|
||||
|
||||
// Console writer
|
||||
consoleWriter := zapcore.AddSync(os.Stdout)
|
||||
|
||||
// Encoder config
|
||||
encoderConfig := zap.NewProductionEncoderConfig()
|
||||
encoderConfig.TimeKey = "timestamp"
|
||||
encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
|
||||
encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder
|
||||
|
||||
// Create cores
|
||||
fileCore := zapcore.NewCore(
|
||||
zapcore.NewJSONEncoder(encoderConfig),
|
||||
fileWriter,
|
||||
level,
|
||||
)
|
||||
|
||||
consoleCore := zapcore.NewCore(
|
||||
zapcore.NewConsoleEncoder(encoderConfig),
|
||||
consoleWriter,
|
||||
level,
|
||||
)
|
||||
|
||||
// Combine cores
|
||||
core := zapcore.NewTee(fileCore, consoleCore)
|
||||
|
||||
// Create logger
|
||||
logger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel))
|
||||
|
||||
return logger, nil
|
||||
}
|
||||
116
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/main.go
vendored
Normal file
116
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/main.go
vendored
Normal file
@@ -0,0 +1,116 @@
|
||||
// OpenAI-compatible chat server using SGLang Go SDK and fasthttp framework
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
_ "net/http/pprof" // Enable pprof endpoints
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"oai_server/config"
|
||||
"oai_server/handlers"
|
||||
"oai_server/logger"
|
||||
"oai_server/service"
|
||||
)
|
||||
|
||||
// Version information (set at build time via ldflags)
|
||||
var (
|
||||
Version = "dev"
|
||||
BuildTime = "unknown"
|
||||
GitCommit = "unknown"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Load configuration
|
||||
cfg := config.Load()
|
||||
|
||||
// Initialize logger
|
||||
appLogger, err := logger.Init(cfg.LogDir, cfg.LogLevel)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to initialize logger: %v", err))
|
||||
}
|
||||
defer appLogger.Sync()
|
||||
|
||||
appLogger.Info("Starting OpenAI-compatible server",
|
||||
zap.String("endpoint", cfg.Endpoint),
|
||||
zap.String("tokenizer", cfg.TokenizerPath),
|
||||
zap.String("port", cfg.Port),
|
||||
)
|
||||
|
||||
// Initialize SGLang service
|
||||
sglangService, err := service.NewSGLangService(cfg.Endpoint, cfg.TokenizerPath)
|
||||
if err != nil {
|
||||
appLogger.Fatal("Failed to create SGLang client", zap.Error(err))
|
||||
}
|
||||
defer sglangService.Close()
|
||||
|
||||
appLogger.Info("SGLang client created successfully")
|
||||
|
||||
// Enable pprof if requested
|
||||
if os.Getenv("PPROF_ENABLED") == "true" {
|
||||
pprofPort := os.Getenv("PPROF_PORT")
|
||||
if pprofPort == "" {
|
||||
pprofPort = "6060"
|
||||
}
|
||||
go func() {
|
||||
pprofAddr := ":" + pprofPort
|
||||
appLogger.Info("Starting pprof server", zap.String("address", pprofAddr))
|
||||
if err := http.ListenAndServe(pprofAddr, nil); err != nil {
|
||||
appLogger.Error("pprof server failed", zap.Error(err))
|
||||
}
|
||||
}()
|
||||
appLogger.Info("pprof enabled", zap.String("port", pprofPort), zap.String("endpoint", fmt.Sprintf("http://localhost:%s/debug/pprof/", pprofPort)))
|
||||
}
|
||||
|
||||
// Initialize handlers
|
||||
healthHandler := handlers.NewHealthHandler(appLogger)
|
||||
modelsHandler := handlers.NewModelsHandler(appLogger, cfg.TokenizerPath)
|
||||
chatHandler := handlers.NewChatHandler(appLogger, sglangService)
|
||||
|
||||
// Setup fasthttp router
|
||||
router := func(ctx *fasthttp.RequestCtx) {
|
||||
path := string(ctx.Path())
|
||||
method := string(ctx.Method())
|
||||
|
||||
switch {
|
||||
case method == "GET" && path == "/health":
|
||||
healthHandler.Check(ctx)
|
||||
case method == "GET" && path == "/v1/models":
|
||||
modelsHandler.List(ctx)
|
||||
case method == "GET" && path == "/get_model_info":
|
||||
modelsHandler.GetModelInfo(ctx)
|
||||
case method == "POST" && path == "/v1/chat/completions":
|
||||
chatHandler.HandleChatCompletion(ctx)
|
||||
case (method == "POST" || method == "PUT") && path == "/generate":
|
||||
chatHandler.HandleGenerate(ctx)
|
||||
default:
|
||||
ctx.Error("Not Found", fasthttp.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// Start server
|
||||
serverAddr := ":" + cfg.Port
|
||||
baseURL := fmt.Sprintf("http://localhost:%s", cfg.Port)
|
||||
|
||||
appLogger.Info("Server starting",
|
||||
zap.String("address", serverAddr),
|
||||
zap.String("base_url", baseURL),
|
||||
)
|
||||
|
||||
// Print available HTTP endpoints (similar to FastAPI startup)
|
||||
appLogger.Info("Available HTTP endpoints:")
|
||||
appLogger.Info(fmt.Sprintf(" GET %s/health", baseURL))
|
||||
appLogger.Info(fmt.Sprintf(" GET %s/v1/models", baseURL))
|
||||
appLogger.Info(fmt.Sprintf(" GET %s/get_model_info", baseURL))
|
||||
appLogger.Info(fmt.Sprintf(" POST %s/v1/chat/completions", baseURL))
|
||||
appLogger.Info(fmt.Sprintf(" POST %s/generate", baseURL))
|
||||
appLogger.Info(fmt.Sprintf("Application startup complete. Listening on %s", baseURL))
|
||||
|
||||
if err := fasthttp.ListenAndServe(serverAddr, router); err != nil {
|
||||
appLogger.Fatal("Server failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
14
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/models/chat.go
vendored
Normal file
14
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/models/chat.go
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
package models
|
||||
|
||||
// ChatRequest represents an OpenAI-compatible chat completion request
|
||||
type ChatRequest struct {
|
||||
Model string `json:"model" binding:"required"`
|
||||
Messages []map[string]string `json:"messages" binding:"required"`
|
||||
Stream bool `json:"stream,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
MaxTokens *int `json:"max_tokens,omitempty"` // OpenAI API standard field
|
||||
MaxCompletionTokens *int `json:"max_completion_tokens,omitempty"` // SGLang-specific field (used by bench_serving.py)
|
||||
Tools []map[string]interface{} `json:"tools,omitempty"`
|
||||
ToolChoice interface{} `json:"tool_choice,omitempty"`
|
||||
}
|
||||
111
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/run.sh
vendored
Executable file
111
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/run.sh
vendored
Executable file
@@ -0,0 +1,111 @@
|
||||
#!/bin/bash
|
||||
|
||||
# OpenAI-compatible server runner
|
||||
# Usage: ./run.sh [tokenizer_path] [endpoint] [port] [--profile] [--pprof-port PORT]
|
||||
#
|
||||
# Options:
|
||||
# --profile Enable pprof profiling (default port: 6060)
|
||||
# --pprof-port PORT Set pprof port (default: 6060, requires --profile)
|
||||
|
||||
# Set library path for Rust FFI library
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BINDINGS_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
LIB_DIR="${BINDINGS_DIR}/lib"
|
||||
|
||||
if [ ! -d "$LIB_DIR" ]; then
|
||||
echo "Error: Library directory not found at $LIB_DIR"
|
||||
echo "Please run 'make lib' first to build and export the library"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get Python LDFLAGS (needed for Rust FFI that depends on Python)
|
||||
PYTHON_LDFLAGS=$(python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "")
|
||||
|
||||
# Set CGO_LDFLAGS to link with the Rust library
|
||||
# Note: -lsgl_model_gateway_go and -ldl are already in the #cgo directive in internal/ffi/client.go
|
||||
# We only need to add the library path (-L) and Python flags
|
||||
export CGO_LDFLAGS="-L${LIB_DIR} ${PYTHON_LDFLAGS}"
|
||||
|
||||
# macOS uses DYLD_LIBRARY_PATH, Linux uses LD_LIBRARY_PATH
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
export DYLD_LIBRARY_PATH="${LIB_DIR}:${DYLD_LIBRARY_PATH}"
|
||||
else
|
||||
export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH}"
|
||||
fi
|
||||
|
||||
# Parse arguments
|
||||
ENABLE_PROFILE=false
|
||||
PPROF_PORT="6060"
|
||||
TOKENIZER_PATH=""
|
||||
ENDPOINT=""
|
||||
PORT=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--profile)
|
||||
ENABLE_PROFILE=true
|
||||
shift
|
||||
;;
|
||||
--pprof-port)
|
||||
ENABLE_PROFILE=true
|
||||
PPROF_PORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$TOKENIZER_PATH" ]]; then
|
||||
TOKENIZER_PATH="$1"
|
||||
elif [[ -z "$ENDPOINT" ]]; then
|
||||
ENDPOINT="$1"
|
||||
elif [[ -z "$PORT" ]]; then
|
||||
PORT="$1"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Default configuration
|
||||
DEFAULT_TOKENIZER_PATH="${SGL_TOKENIZER_PATH:-../tokenizer}"
|
||||
DEFAULT_ENDPOINT="${SGL_GRPC_ENDPOINT:-grpc://localhost:20000}"
|
||||
DEFAULT_PORT="${PORT:-8080}"
|
||||
|
||||
TOKENIZER_PATH="${TOKENIZER_PATH:-${DEFAULT_TOKENIZER_PATH}}"
|
||||
ENDPOINT="${ENDPOINT:-${DEFAULT_ENDPOINT}}"
|
||||
PORT="${PORT:-${DEFAULT_PORT}}"
|
||||
|
||||
echo "Running OpenAI-compatible server..."
|
||||
echo "Library path: ${LIB_DIR}"
|
||||
echo "Tokenizer: $TOKENIZER_PATH"
|
||||
echo "Endpoint: $ENDPOINT"
|
||||
echo "Port: $PORT"
|
||||
echo "Client Mode: gRPC (default)"
|
||||
echo "FFI Postprocessing: ENABLED (normal mode)"
|
||||
echo "FFI Preprocessing: ENABLED (normal mode)"
|
||||
if [[ "$ENABLE_PROFILE" == "true" ]]; then
|
||||
echo "Profiling: enabled (port: $PPROF_PORT)"
|
||||
echo " pprof endpoint: http://localhost:$PPROF_PORT/debug/pprof/"
|
||||
export PPROF_ENABLED=true
|
||||
export PPROF_PORT="$PPROF_PORT"
|
||||
else
|
||||
echo "Profiling: disabled"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Change to script directory
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||
|
||||
# Ensure Go module is properly initialized
|
||||
if [ ! -f "go.mod" ]; then
|
||||
echo "Error: go.mod not found in $(pwd)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Ensure Go modules are enabled
|
||||
export GO111MODULE=on
|
||||
|
||||
# Sync Go module dependencies
|
||||
echo "Syncing Go module dependencies..."
|
||||
go mod tidy
|
||||
|
||||
# Run the server (use ./main.go to ensure module context is correct)
|
||||
SGL_TOKENIZER_PATH="$TOKENIZER_PATH" SGL_GRPC_ENDPOINT="$ENDPOINT" PORT="$PORT" go run ./main.go
|
||||
554
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/analyze_tpot.sh
vendored
Executable file
554
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/analyze_tpot.sh
vendored
Executable file
@@ -0,0 +1,554 @@
|
||||
#!/bin/bash
|
||||
|
||||
# TPOT performance bottleneck analysis script
|
||||
# Specifically designed to analyze why Go Router is twice as slow as Rust Router
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/analyze_tpot.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# --duration SECONDS CPU profile duration (default: 60)
|
||||
# --requests NUM Number of requests (default: 100)
|
||||
# --concurrency NUM Concurrency level (default: 20)
|
||||
# --pprof-port PORT pprof port (default: 6060)
|
||||
# --server-url URL Server URL (default: http://localhost:8080)
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
PROFILE_DIR="${PROJECT_ROOT}/profiles"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
OUTPUT_DIR="${PROFILE_DIR}/tpot_analysis_${TIMESTAMP}"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Default values
|
||||
DURATION=${DURATION:-60}
|
||||
NUM_REQUESTS=${NUM_REQUESTS:-100}
|
||||
CONCURRENCY=${CONCURRENCY:-20}
|
||||
PPROF_PORT=${PPROF_PORT:-6060}
|
||||
SERVER_URL=${SERVER_URL:-http://localhost:8080}
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--duration)
|
||||
DURATION="$2"
|
||||
shift 2
|
||||
;;
|
||||
--requests)
|
||||
NUM_REQUESTS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--concurrency)
|
||||
CONCURRENCY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--pprof-port)
|
||||
PPROF_PORT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--server-url)
|
||||
SERVER_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Check for graphviz (optional, needed for some pprof visualizations)
|
||||
HAS_GRAPHVIZ=false
|
||||
if command -v dot >/dev/null 2>&1; then
|
||||
HAS_GRAPHVIZ=true
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}TPOT Performance Bottleneck Analysis${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
echo " Duration: ${DURATION}s"
|
||||
echo " Requests: $NUM_REQUESTS"
|
||||
echo " Concurrency: $CONCURRENCY"
|
||||
echo " Server URL: $SERVER_URL"
|
||||
echo " pprof Port: $PPROF_PORT"
|
||||
echo " Output Dir: $OUTPUT_DIR"
|
||||
if [ "$HAS_GRAPHVIZ" = "false" ]; then
|
||||
echo ""
|
||||
echo -e "${YELLOW}Note: graphviz not found. Some pprof visualizations may not work.${NC}"
|
||||
echo -e "${YELLOW}To install graphviz:${NC}"
|
||||
echo -e "${YELLOW} macOS: brew install graphviz${NC}"
|
||||
echo -e "${YELLOW} Ubuntu: sudo apt-get install graphviz${NC}"
|
||||
echo -e "${YELLOW} CentOS: sudo yum install graphviz${NC}"
|
||||
echo -e "${YELLOW}Text reports will still be generated without graphviz.${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check if server is running
|
||||
echo -e "${YELLOW}[Check] Verifying server is running...${NC}"
|
||||
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
|
||||
echo -e "${RED}Error: Server not responding at ${SERVER_URL}${NC}"
|
||||
echo ""
|
||||
echo "Please start the server first with profiling enabled:"
|
||||
echo " ./run.sh --profile --pprof-port $PPROF_PORT"
|
||||
echo " or"
|
||||
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT make run"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${GREEN}✓ Server is running${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if pprof is enabled
|
||||
echo -e "${YELLOW}[Check] Verifying pprof is enabled...${NC}"
|
||||
if ! curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
|
||||
echo -e "${RED}Error: pprof not accessible at http://localhost:${PPROF_PORT}/debug/pprof/${NC}"
|
||||
echo ""
|
||||
echo "Please start the server with profiling enabled:"
|
||||
echo " ./run.sh --profile --pprof-port $PPROF_PORT"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${GREEN}✓ pprof is enabled${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 1: Collect baseline profiles
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 1/8] Collecting baseline profiles...${NC}"
|
||||
|
||||
# Baseline memory
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/heap_before.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
|
||||
|
||||
# Baseline goroutine
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/goroutine_before.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" > /dev/null 2>&1 || true
|
||||
|
||||
echo -e "${GREEN}✓ Baseline profiles collected${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 2: Start CPU profile collection
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 2/8] Starting CPU profile collection (${DURATION}s)...${NC}"
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}" &
|
||||
CPU_PID=$!
|
||||
sleep 2
|
||||
echo -e "${GREEN}✓ CPU profile collection started${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 3: Run load test with streaming requests
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 3/8] Running load test ($NUM_REQUESTS streaming requests, concurrency=$CONCURRENCY)...${NC}"
|
||||
|
||||
# Function to run a single streaming request
|
||||
run_streaming_request() {
|
||||
local request_id=$1
|
||||
local start_time=$(date +%s)
|
||||
local start_nanos=$(date +%N 2>/dev/null || echo "000000000")
|
||||
|
||||
curl -N -s -X POST "${SERVER_URL}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"model\": \"default\",
|
||||
\"messages\": [{\"role\": \"user\", \"content\": \"Write a 500-word story with character dialogue and scene descriptions\"}],
|
||||
\"stream\": true,
|
||||
\"max_tokens\": 300,
|
||||
\"temperature\": 0.7
|
||||
}" > /dev/null
|
||||
|
||||
local end_time=$(date +%s)
|
||||
local end_nanos=$(date +%N 2>/dev/null || echo "000000000")
|
||||
local duration=$((end_time - start_time))
|
||||
echo "$duration" >> "${OUTPUT_DIR}/request_times.txt"
|
||||
}
|
||||
|
||||
# Run requests with controlled concurrency
|
||||
# Use a temporary file to track job PIDs to avoid conflicts with CPU_PID
|
||||
JOB_PIDS_FILE="${OUTPUT_DIR}/.job_pids_$$"
|
||||
> "$JOB_PIDS_FILE"
|
||||
|
||||
for i in $(seq 1 $NUM_REQUESTS); do
|
||||
# Wait if we've reached concurrency limit
|
||||
while [ $(wc -l < "$JOB_PIDS_FILE" 2>/dev/null || echo 0) -ge $CONCURRENCY ]; do
|
||||
# Check and remove completed jobs
|
||||
while IFS= read -r pid; do
|
||||
if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then
|
||||
# Process completed, remove from file
|
||||
grep -v "^${pid}$" "$JOB_PIDS_FILE" > "${JOB_PIDS_FILE}.tmp" && \
|
||||
mv "${JOB_PIDS_FILE}.tmp" "$JOB_PIDS_FILE" || true
|
||||
fi
|
||||
done < "$JOB_PIDS_FILE"
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
# Start new request
|
||||
run_streaming_request $i &
|
||||
echo $! >> "$JOB_PIDS_FILE"
|
||||
|
||||
# Progress indicator
|
||||
if [ $((i % 10)) -eq 0 ]; then
|
||||
echo " Progress: $i/$NUM_REQUESTS requests sent..."
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait for all remaining jobs (excluding CPU_PID)
|
||||
while IFS= read -r pid; do
|
||||
if [ -n "$pid" ] && [ "$pid" != "$CPU_PID" ]; then
|
||||
wait "$pid" 2>/dev/null || true
|
||||
fi
|
||||
done < "$JOB_PIDS_FILE"
|
||||
|
||||
# Clean up
|
||||
rm -f "$JOB_PIDS_FILE" "${JOB_PIDS_FILE}.tmp" 2>/dev/null || true
|
||||
|
||||
echo -e "${GREEN}✓ Load test completed${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 4: Wait for CPU profile to complete
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 4/8] Waiting for CPU profile to complete...${NC}"
|
||||
# Wait for the process, but handle the case where it might have already completed
|
||||
if kill -0 $CPU_PID 2>/dev/null; then
|
||||
wait $CPU_PID 2>/dev/null || true
|
||||
else
|
||||
# Process already completed, just wait a bit to ensure file is written
|
||||
sleep 1
|
||||
fi
|
||||
echo -e "${GREEN}✓ CPU profile collection completed${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 5: Collect final profiles
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 5/8] Collecting final profiles...${NC}"
|
||||
|
||||
# Final memory
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/heap_after.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
|
||||
|
||||
# Final goroutine
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/goroutine_after.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" > /dev/null 2>&1 || true
|
||||
|
||||
# Mutex profile
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/mutex.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/mutex" > /dev/null 2>&1 || true
|
||||
|
||||
# Block profile
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/block.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/block" > /dev/null 2>&1 || true
|
||||
|
||||
echo -e "${GREEN}✓ Final profiles collected${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 6: Generate analysis reports
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 6/8] Generating analysis reports...${NC}"
|
||||
|
||||
# CPU analysis
|
||||
echo " Generating CPU reports..."
|
||||
go tool pprof -top -cum "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/01_cpu_top_cum.txt" 2>&1 || true
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/02_cpu_top_flat.txt" 2>&1 || true
|
||||
|
||||
# Memory analysis
|
||||
echo " Generating memory reports..."
|
||||
if [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
|
||||
go tool pprof -top -alloc_space "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/03_memory_alloc_space.txt" 2>&1 || true
|
||||
go tool pprof -top -alloc_objects "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/04_memory_alloc_objects.txt" 2>&1 || true
|
||||
go tool pprof -top -inuse_space "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/05_memory_inuse_space.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# Memory growth
|
||||
if [ -f "${OUTPUT_DIR}/heap_before.pb.gz" ] && [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
|
||||
go tool pprof -top -base="${OUTPUT_DIR}/heap_before.pb.gz" \
|
||||
"${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/06_memory_growth.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# FFI/CGO analysis
|
||||
echo " Analyzing FFI/CGO calls..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
|
||||
grep -iE "(block_on|CGO|FFI|ffi|runtime\.cgo|_Cfunc)" > "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt" || \
|
||||
echo "No FFI/CGO related functions found" > "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
|
||||
|
||||
# JSON serialization analysis
|
||||
echo " Analyzing JSON serialization..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
|
||||
grep -iE "(json|Marshal|Unmarshal|Encode|Decode|sonic|jsoniter)" > "${OUTPUT_DIR}/08_json_analysis.txt" || \
|
||||
echo "No JSON related functions found" > "${OUTPUT_DIR}/08_json_analysis.txt"
|
||||
|
||||
# Goroutine analysis
|
||||
if [ -f "${OUTPUT_DIR}/goroutine_after.pb.gz" ]; then
|
||||
echo " Analyzing goroutines..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/goroutine_after.pb.gz" > "${OUTPUT_DIR}/09_goroutine_analysis.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# Mutex analysis
|
||||
if [ -f "${OUTPUT_DIR}/mutex.pb.gz" ]; then
|
||||
echo " Analyzing mutex contention..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/mutex.pb.gz" > "${OUTPUT_DIR}/10_mutex_analysis.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# Block analysis
|
||||
if [ -f "${OUTPUT_DIR}/block.pb.gz" ]; then
|
||||
echo " Analyzing blocking operations..."
|
||||
go tool pprof -top "${OUTPUT_DIR}/block.pb.gz" > "${OUTPUT_DIR}/11_block_analysis.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# Request timing statistics
|
||||
if [ -f "${OUTPUT_DIR}/request_times.txt" ] && [ -s "${OUTPUT_DIR}/request_times.txt" ]; then
|
||||
echo " Calculating request timing statistics..."
|
||||
{
|
||||
echo "Request Timing Statistics"
|
||||
echo "========================"
|
||||
echo ""
|
||||
echo "Total requests: $(wc -l < "${OUTPUT_DIR}/request_times.txt" | tr -d ' ')"
|
||||
echo ""
|
||||
awk '{
|
||||
sum+=$1
|
||||
sumsq+=$1*$1
|
||||
if(NR==1 || $1<min) min=$1
|
||||
if(NR==1 || $1>max) max=$1
|
||||
} END {
|
||||
if(NR > 0) {
|
||||
mean=sum/NR
|
||||
variance=(sumsq/NR - mean*mean)
|
||||
stddev=sqrt(variance)
|
||||
print "Min: " min "s"
|
||||
print "Max: " max "s"
|
||||
print "Mean: " mean "s"
|
||||
print "StdDev: " stddev "s"
|
||||
}
|
||||
}' "${OUTPUT_DIR}/request_times.txt"
|
||||
} > "${OUTPUT_DIR}/12_request_timing.txt"
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✓ Analysis reports generated${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 7: Generate summary report
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 7/8] Generating summary report...${NC}"
|
||||
|
||||
SUMMARY_FILE="${OUTPUT_DIR}/00_SUMMARY.md"
|
||||
cat > "$SUMMARY_FILE" <<EOF
|
||||
# TPOT Performance Analysis Summary
|
||||
|
||||
**Analysis Date:** $(date)
|
||||
**Duration:** ${DURATION}s
|
||||
**Requests:** $NUM_REQUESTS
|
||||
**Concurrency:** $CONCURRENCY
|
||||
|
||||
## Key Findings
|
||||
|
||||
### 1. CPU Hotspots (Top 10 Cumulative Time)
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/01_cpu_top_cum.txt" | tail -10)
|
||||
\`\`\`
|
||||
|
||||
### 2. CPU Hotspots (Top 10 Flat Time)
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/02_cpu_top_flat.txt" | tail -10)
|
||||
\`\`\`
|
||||
|
||||
### 3. FFI/CGO Overhead
|
||||
|
||||
\`\`\`
|
||||
$(cat "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt")
|
||||
\`\`\`
|
||||
|
||||
### 4. JSON Serialization Overhead
|
||||
|
||||
\`\`\`
|
||||
$(cat "${OUTPUT_DIR}/08_json_analysis.txt")
|
||||
\`\`\`
|
||||
|
||||
### 5. Memory Allocation (Top 10 by Space)
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/03_memory_alloc_space.txt" | tail -10)
|
||||
\`\`\`
|
||||
|
||||
### 6. Memory Allocation (Top 10 by Objects)
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/04_memory_alloc_objects.txt" | tail -10)
|
||||
\`\`\`
|
||||
|
||||
### 7. Mutex Contention
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/10_mutex_analysis.txt" | tail -10 2>/dev/null || echo "No significant mutex contention detected")
|
||||
\`\`\`
|
||||
|
||||
### 8. Blocking Operations
|
||||
|
||||
\`\`\`
|
||||
$(head -15 "${OUTPUT_DIR}/11_block_analysis.txt" | tail -10 2>/dev/null || echo "No significant blocking detected")
|
||||
\`\`\`
|
||||
|
||||
## Performance Bottlenecks Identified
|
||||
|
||||
### High Priority Issues
|
||||
|
||||
1. **FFI/CGO Overhead**
|
||||
- Check: \`cat ${OUTPUT_DIR}/07_ffi_cgo_analysis.txt\`
|
||||
- Impact: FFI calls add overhead compared to native Rust code
|
||||
- Recommendation: Minimize FFI calls, batch operations
|
||||
|
||||
2. **JSON Serialization**
|
||||
- Check: \`cat ${OUTPUT_DIR}/08_json_analysis.txt\`
|
||||
- Impact: JSON marshaling/unmarshaling can be expensive
|
||||
- Recommendation: Use faster JSON library (jsoniter), reduce serialization frequency
|
||||
|
||||
3. **Memory Allocations**
|
||||
- Check: \`cat ${OUTPUT_DIR}/03_memory_alloc_space.txt\`
|
||||
- Impact: Frequent allocations cause GC pressure
|
||||
- Recommendation: Use object pools, pre-allocate buffers
|
||||
|
||||
### Medium Priority Issues
|
||||
|
||||
4. **Goroutine Overhead**
|
||||
- Check: \`cat ${OUTPUT_DIR}/09_goroutine_analysis.txt\`
|
||||
- Impact: Too many goroutines can cause scheduling overhead
|
||||
- Recommendation: Limit goroutine count, use worker pools
|
||||
|
||||
5. **Lock Contention**
|
||||
- Check: \`cat ${OUTPUT_DIR}/10_mutex_analysis.txt\`
|
||||
- Impact: Lock contention reduces parallelism
|
||||
- Recommendation: Reduce lock granularity, use lock-free structures
|
||||
|
||||
## Comparison with Rust Router
|
||||
|
||||
### Expected Differences
|
||||
|
||||
1. **FFI Overhead**: Go → Rust FFI calls add ~100-500ns per call
|
||||
2. **GC Overhead**: Go's GC can cause pauses (usually <1ms)
|
||||
3. **JSON Library**: Go's standard library is slower than Rust's serde
|
||||
4. **Memory Layout**: Go's GC affects cache locality
|
||||
|
||||
### Optimization Opportunities
|
||||
|
||||
1. **Reduce FFI Calls**
|
||||
- Batch token processing
|
||||
- Use async FFI (if possible)
|
||||
- Cache frequently used FFI results
|
||||
|
||||
2. **Optimize JSON**
|
||||
- Use jsoniter (already implemented)
|
||||
- Pre-allocate JSON buffers
|
||||
- Reduce serialization frequency
|
||||
|
||||
3. **Memory Management**
|
||||
- Use sync.Pool for frequently allocated objects
|
||||
- Pre-allocate slices with known capacity
|
||||
- Avoid unnecessary string copies
|
||||
|
||||
4. **Concurrency**
|
||||
- Use worker pools instead of spawning goroutines per request
|
||||
- Limit concurrent FFI calls
|
||||
- Use channels efficiently
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Review detailed reports in this directory
|
||||
2. Use interactive pprof: \`go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz\`
|
||||
3. Compare with Rust router profiles (if available)
|
||||
4. Implement optimizations based on findings
|
||||
5. Re-run analysis to measure improvements
|
||||
|
||||
## Files Generated
|
||||
|
||||
- \`00_SUMMARY.md\` - This summary
|
||||
- \`01_cpu_top_cum.txt\` - CPU top functions (cumulative)
|
||||
- \`02_cpu_top_flat.txt\` - CPU top functions (flat)
|
||||
- \`03_memory_alloc_space.txt\` - Memory allocation by space
|
||||
- \`04_memory_alloc_objects.txt\` - Memory allocation by objects
|
||||
- \`05_memory_inuse_space.txt\` - Memory in use by space
|
||||
- \`06_memory_growth.txt\` - Memory growth during test
|
||||
- \`07_ffi_cgo_analysis.txt\` - FFI/CGO overhead analysis
|
||||
- \`08_json_analysis.txt\` - JSON serialization analysis
|
||||
- \`09_goroutine_analysis.txt\` - Goroutine analysis
|
||||
- \`10_mutex_analysis.txt\` - Mutex contention analysis
|
||||
- \`11_block_analysis.txt\` - Blocking operations analysis
|
||||
- \`12_request_timing.txt\` - Request timing statistics
|
||||
- \`*.pb.gz\` - Raw profile files for interactive analysis
|
||||
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}✓ Summary report generated${NC}"
|
||||
echo ""
|
||||
|
||||
# ============================================
|
||||
# Step 8: Display summary
|
||||
# ============================================
|
||||
echo -e "${GREEN}[Step 8/8] Analysis Complete!${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}Summary${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Top CPU Hotspots (Cumulative):${NC}"
|
||||
head -12 "${OUTPUT_DIR}/01_cpu_top_cum.txt" | tail -10
|
||||
echo ""
|
||||
echo -e "${YELLOW}FFI/CGO Overhead:${NC}"
|
||||
cat "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
|
||||
echo ""
|
||||
echo -e "${YELLOW}JSON Serialization Overhead:${NC}"
|
||||
cat "${OUTPUT_DIR}/08_json_analysis.txt"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Top Memory Allocations:${NC}"
|
||||
head -12 "${OUTPUT_DIR}/03_memory_alloc_space.txt" | tail -10
|
||||
echo ""
|
||||
if [ -f "${OUTPUT_DIR}/12_request_timing.txt" ]; then
|
||||
echo -e "${YELLOW}Request Timing:${NC}"
|
||||
cat "${OUTPUT_DIR}/12_request_timing.txt"
|
||||
echo ""
|
||||
fi
|
||||
echo -e "${GREEN}========================================${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}Detailed Reports:${NC}"
|
||||
echo " Summary: cat ${OUTPUT_DIR}/00_SUMMARY.md"
|
||||
echo " CPU (cum): cat ${OUTPUT_DIR}/01_cpu_top_cum.txt"
|
||||
echo " CPU (flat): cat ${OUTPUT_DIR}/02_cpu_top_flat.txt"
|
||||
echo " FFI/CGO: cat ${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
|
||||
echo " JSON: cat ${OUTPUT_DIR}/08_json_analysis.txt"
|
||||
echo " Memory: cat ${OUTPUT_DIR}/03_memory_alloc_space.txt"
|
||||
echo ""
|
||||
echo -e "${BLUE}Interactive Analysis:${NC}"
|
||||
echo " Run: go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz"
|
||||
echo " Then visit:"
|
||||
echo " - http://localhost:8081/ui/flamegraph (Flame Graph - no graphviz needed)"
|
||||
echo " - http://localhost:8081/ui/top (Top Functions - no graphviz needed)"
|
||||
if [ "$HAS_GRAPHVIZ" = "true" ]; then
|
||||
echo " - http://localhost:8081/ui/graph (Call Graph - requires graphviz)"
|
||||
else
|
||||
echo " - http://localhost:8081/ui/graph (Call Graph - requires graphviz, not available)"
|
||||
fi
|
||||
echo ""
|
||||
if [ "$HAS_GRAPHVIZ" = "false" ]; then
|
||||
echo -e "${YELLOW}Note: Install graphviz to enable call graph visualization:${NC}"
|
||||
echo -e "${YELLOW} macOS: brew install graphviz${NC}"
|
||||
echo -e "${YELLOW} Ubuntu: sudo apt-get install graphviz${NC}"
|
||||
echo -e "${YELLOW} CentOS: sudo yum install graphviz${NC}"
|
||||
echo ""
|
||||
fi
|
||||
echo -e "${GREEN}All files saved to: ${OUTPUT_DIR}${NC}"
|
||||
echo ""
|
||||
215
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/pprof_analysis.sh
vendored
Executable file
215
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/pprof_analysis.sh
vendored
Executable file
@@ -0,0 +1,215 @@
|
||||
#!/bin/bash
|
||||
|
||||
# pprof performance analysis script
|
||||
# Used to analyze performance bottlenecks of Go OpenAI server
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Configuration
|
||||
PPROF_PORT=${PPROF_PORT:-6060}
|
||||
SERVER_PORT=${SERVER_PORT:-8080}
|
||||
DURATION=${DURATION:-60} # Performance test duration (seconds)
|
||||
OUTPUT_DIR="./pprof_results"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
echo "=========================================="
|
||||
echo "pprof Performance Analysis Tool"
|
||||
echo "=========================================="
|
||||
echo "PPROF_PORT: $PPROF_PORT"
|
||||
echo "SERVER_PORT: $SERVER_PORT"
|
||||
echo "DURATION: ${DURATION}s"
|
||||
echo "OUTPUT_DIR: $OUTPUT_DIR"
|
||||
echo ""
|
||||
|
||||
# Check if go tool pprof is available
|
||||
if ! command -v go &> /dev/null; then
|
||||
echo "Error: go command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if server is running
|
||||
check_server() {
|
||||
if curl -s "http://localhost:${SERVER_PORT}/health" > /dev/null 2>&1; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if pprof is available
|
||||
check_pprof() {
|
||||
if curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Start server (if not running)
|
||||
if ! check_server; then
|
||||
echo "Server not running, please start the server first:"
|
||||
echo " export PPROF_ENABLED=true"
|
||||
echo " export PPROF_PORT=$PPROF_PORT"
|
||||
echo " ./oai_server"
|
||||
echo ""
|
||||
echo "Or use the following command to start:"
|
||||
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT ./oai_server"
|
||||
echo ""
|
||||
read -p "Start server now? (y/n) " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Starting server..."
|
||||
PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT ./oai_server &
|
||||
SERVER_PID=$!
|
||||
echo "Server PID: $SERVER_PID"
|
||||
|
||||
# Wait for server to start
|
||||
echo "Waiting for server to start..."
|
||||
for i in {1..30}; do
|
||||
if check_server; then
|
||||
echo "Server started"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if ! check_server; then
|
||||
echo "Error: Server failed to start"
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if pprof is available
|
||||
if ! check_pprof; then
|
||||
echo "Error: pprof not enabled. Please set environment variables:"
|
||||
echo " export PPROF_ENABLED=true"
|
||||
echo " export PPROF_PORT=$PPROF_PORT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting to collect performance data..."
|
||||
echo ""
|
||||
|
||||
# 1. CPU Profile (30 seconds)
|
||||
echo "[1/6] Collecting CPU Profile (30 seconds)..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30" &
|
||||
CPU_PID=$!
|
||||
|
||||
# 2. Collect Heap Profile simultaneously
|
||||
echo "[2/6] Collecting Heap Profile..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" &
|
||||
HEAP_PID=$!
|
||||
|
||||
# 3. Collect Goroutine Profile
|
||||
echo "[3/6] Collecting Goroutine Profile..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/goroutine_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" &
|
||||
GOROUTINE_PID=$!
|
||||
|
||||
# 4. Collect Mutex Profile
|
||||
echo "[4/6] Collecting Mutex Profile..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/mutex_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/mutex" &
|
||||
MUTEX_PID=$!
|
||||
|
||||
# 5. Collect Block Profile
|
||||
echo "[5/6] Collecting Block Profile..."
|
||||
go tool pprof -proto -output="$OUTPUT_DIR/block_${TIMESTAMP}.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/block" &
|
||||
BLOCK_PID=$!
|
||||
|
||||
# 6. Run performance test (during CPU profile collection)
|
||||
echo "[6/6] Running performance test..."
|
||||
echo "Tip: Please use your performance testing tool (curl, ab, wrk, etc.) to send requests to the server"
|
||||
echo " CPU profile will collect 30 seconds of performance data"
|
||||
echo ""
|
||||
|
||||
# Wait for CPU profile to complete
|
||||
wait $CPU_PID
|
||||
echo "CPU Profile collection completed"
|
||||
|
||||
# Wait for other profiles
|
||||
wait $HEAP_PID
|
||||
wait $GOROUTINE_PID
|
||||
wait $MUTEX_PID
|
||||
wait $BLOCK_PID
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Performance data collection completed!"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Generated analysis files:"
|
||||
ls -lh "$OUTPUT_DIR"/*_${TIMESTAMP}.* 2>/dev/null || true
|
||||
echo ""
|
||||
|
||||
# Generate analysis report
|
||||
echo "Generating analysis report..."
|
||||
echo ""
|
||||
|
||||
# CPU Top 20
|
||||
echo "=== CPU Top 20 (sorted by flat time) ===" > "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top -cum "$OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
|
||||
# Heap Top 20
|
||||
echo "=== Heap Top 20 (sorted by allocation size) ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top "$OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
|
||||
# Goroutine statistics
|
||||
echo "=== Goroutine Statistics ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top "$OUTPUT_DIR/goroutine_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
|
||||
# Mutex statistics
|
||||
echo "=== Mutex Wait Time ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top "$OUTPUT_DIR/mutex_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
|
||||
# Block statistics
|
||||
echo "=== Block Wait Time ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
go tool pprof -top "$OUTPUT_DIR/block_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
|
||||
|
||||
echo "Analysis report saved to: $OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
echo ""
|
||||
|
||||
# Display key information
|
||||
echo "=========================================="
|
||||
echo "Key Performance Metrics Summary"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "View detailed report:"
|
||||
echo " cat $OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
|
||||
echo ""
|
||||
echo "Interactive CPU Profile view:"
|
||||
echo " go tool pprof $OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz"
|
||||
echo ""
|
||||
echo "Interactive Heap Profile view:"
|
||||
echo " go tool pprof $OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz"
|
||||
echo ""
|
||||
echo "Generate flame graph (requires go-torch or pprof):"
|
||||
echo " go tool pprof -http=:8080 $OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz"
|
||||
echo ""
|
||||
|
||||
# If server was started, ask if it should be closed
|
||||
if [ -n "$SERVER_PID" ]; then
|
||||
read -p "Close server? (y/n) " -n 1 -r
|
||||
echo
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
echo "Server closed"
|
||||
fi
|
||||
fi
|
||||
52
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/pprof_quick.sh
vendored
Executable file
52
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/pprof_quick.sh
vendored
Executable file
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Quick pprof analysis script
|
||||
# Collects 30-second CPU profile and immediately displays top results
|
||||
|
||||
set -e
|
||||
|
||||
PPROF_PORT=${PPROF_PORT:-6060}
|
||||
DURATION=${DURATION:-30}
|
||||
|
||||
echo "=========================================="
|
||||
echo "Quick pprof Analysis"
|
||||
echo "=========================================="
|
||||
echo "PPROF_PORT: $PPROF_PORT"
|
||||
echo "DURATION: ${DURATION}s"
|
||||
echo ""
|
||||
echo "Tip: During data collection, please send requests to the server"
|
||||
echo " You can use: ./pprof_test.sh"
|
||||
echo ""
|
||||
|
||||
# Check if pprof is available
|
||||
if ! curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
|
||||
echo "Error: pprof not enabled. Please set environment variables:"
|
||||
echo " export PPROF_ENABLED=true"
|
||||
echo " export PPROF_PORT=$PPROF_PORT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting to collect CPU Profile (${DURATION} seconds)..."
|
||||
echo ""
|
||||
|
||||
# Collect CPU profile and directly display top results
|
||||
go tool pprof -top -cum "http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Analysis Complete"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "More analysis options:"
|
||||
echo " # Interactive view"
|
||||
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30"
|
||||
echo ""
|
||||
echo " # View heap memory"
|
||||
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/heap"
|
||||
echo ""
|
||||
echo " # View goroutines"
|
||||
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/goroutine"
|
||||
echo ""
|
||||
echo " # Generate Web UI"
|
||||
echo " go tool pprof -http=:8080 http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30"
|
||||
echo ""
|
||||
87
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/pprof_test.sh
vendored
Executable file
87
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/pprof_test.sh
vendored
Executable file
@@ -0,0 +1,87 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Simple performance test script for sending requests while collecting pprof data
|
||||
|
||||
set -e
|
||||
|
||||
SERVER_URL=${SERVER_URL:-"http://localhost:8080"}
|
||||
DURATION=${DURATION:-30} # Test duration (seconds)
|
||||
CONCURRENT=${CONCURRENT:-1} # Number of concurrent requests
|
||||
|
||||
echo "=========================================="
|
||||
echo "Performance Test Script"
|
||||
echo "=========================================="
|
||||
echo "SERVER_URL: $SERVER_URL"
|
||||
echo "DURATION: ${DURATION}s"
|
||||
echo "CONCURRENT: $CONCURRENT"
|
||||
echo ""
|
||||
|
||||
# Test request JSON
|
||||
TEST_REQUEST='{
|
||||
"model": "default",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, how are you?"}
|
||||
],
|
||||
"stream": true,
|
||||
"max_tokens": 100
|
||||
}'
|
||||
|
||||
# Check if server is available
|
||||
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
|
||||
echo "Error: Server not available (${SERVER_URL}/health)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting to send test requests..."
|
||||
echo ""
|
||||
|
||||
# Function to send streaming request
|
||||
send_stream_request() {
|
||||
local request_num=$1
|
||||
local start_time=$(date +%s.%N)
|
||||
|
||||
curl -s -N -X POST "${SERVER_URL}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$TEST_REQUEST" \
|
||||
> /dev/null 2>&1
|
||||
|
||||
local end_time=$(date +%s.%N)
|
||||
local duration=$(echo "$end_time - $start_time" | bc)
|
||||
echo "Request $request_num completed, duration: ${duration}s"
|
||||
}
|
||||
|
||||
# Send requests concurrently
|
||||
if [ "$CONCURRENT" -eq 1 ]; then
|
||||
# Single-threaded mode: continuously send requests
|
||||
end_time=$(($(date +%s) + DURATION))
|
||||
request_count=0
|
||||
|
||||
while [ $(date +%s) -lt $end_time ]; do
|
||||
request_count=$((request_count + 1))
|
||||
send_stream_request $request_count
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Test completed, sent $request_count requests"
|
||||
else
|
||||
# Multi-threaded mode: send requests concurrently
|
||||
end_time=$(($(date +%s) + DURATION))
|
||||
request_count=0
|
||||
|
||||
while [ $(date +%s) -lt $end_time ]; do
|
||||
# Start concurrent requests
|
||||
for i in $(seq 1 $CONCURRENT); do
|
||||
request_count=$((request_count + 1))
|
||||
send_stream_request $request_count &
|
||||
done
|
||||
|
||||
# Wait for all requests to complete
|
||||
wait
|
||||
|
||||
# Brief rest to avoid overload
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Test completed, sent $request_count requests"
|
||||
fi
|
||||
140
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/profile_tpot.sh
vendored
Executable file
140
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/scripts/profile_tpot.sh
vendored
Executable file
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
|
||||
# TPOT performance analysis script
|
||||
# Quickly collect and analyze TPOT-related performance data
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
PROFILE_DIR="${PROJECT_ROOT}/profiles"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
OUTPUT_DIR="${PROFILE_DIR}/${TIMESTAMP}"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Default values
|
||||
PPROF_PORT=${PPROF_PORT:-6060}
|
||||
SERVER_URL=${SERVER_URL:-http://localhost:8080}
|
||||
DURATION=${DURATION:-30}
|
||||
NUM_REQUESTS=${NUM_REQUESTS:-20}
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
echo -e "${GREEN}TPOT Performance Analysis${NC}"
|
||||
echo "=========================="
|
||||
echo "Profile directory: $OUTPUT_DIR"
|
||||
echo "Duration: ${DURATION}s"
|
||||
echo "Requests: $NUM_REQUESTS"
|
||||
echo ""
|
||||
|
||||
# Check if server is running
|
||||
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
|
||||
echo -e "${YELLOW}Warning: Server not responding at ${SERVER_URL}${NC}"
|
||||
echo "Please start the server first with profiling enabled:"
|
||||
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT make run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Collect baseline memory
|
||||
echo -e "${GREEN}[1/5] Collecting baseline memory profile...${NC}"
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/heap_before.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
|
||||
|
||||
# Start CPU profile collection in background
|
||||
echo -e "${GREEN}[2/5] Starting CPU profile collection (${DURATION}s)...${NC}"
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}" &
|
||||
CPU_PID=$!
|
||||
|
||||
# Wait a bit for profile to start
|
||||
sleep 2
|
||||
|
||||
# Run load test
|
||||
echo -e "${GREEN}[3/5] Running load test ($NUM_REQUESTS requests)...${NC}"
|
||||
for i in $(seq 1 $NUM_REQUESTS); do
|
||||
curl -N -s -X POST "${SERVER_URL}/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"model\": \"default\",
|
||||
\"messages\": [{\"role\": \"user\", \"content\": \"Write a story\"}],
|
||||
\"stream\": true,
|
||||
\"max_tokens\": 200
|
||||
}" > /dev/null &
|
||||
|
||||
# Limit concurrency
|
||||
if [ $((i % 5)) -eq 0 ]; then
|
||||
wait
|
||||
fi
|
||||
done
|
||||
wait
|
||||
|
||||
# Wait for CPU profile to complete
|
||||
echo -e "${GREEN}[4/5] Waiting for CPU profile to complete...${NC}"
|
||||
# Wait for the CPU profile process, but handle the case where it's not a child process
|
||||
if kill -0 $CPU_PID 2>/dev/null; then
|
||||
# Process is still running, wait for it
|
||||
while kill -0 $CPU_PID 2>/dev/null; do
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
# Process already completed or not found, just wait a bit
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Collect final memory
|
||||
echo -e "${GREEN}[5/5] Collecting final memory profile...${NC}"
|
||||
go tool pprof -proto -output="${OUTPUT_DIR}/heap_after.pb.gz" \
|
||||
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
|
||||
|
||||
# Generate reports
|
||||
echo ""
|
||||
echo -e "${GREEN}Generating reports...${NC}"
|
||||
|
||||
# CPU top (cumulative)
|
||||
go tool pprof -top -cum "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/cpu_top_cum.txt" 2>&1 || true
|
||||
|
||||
# CPU top (flat)
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/cpu_top_flat.txt" 2>&1 || true
|
||||
|
||||
# Memory growth
|
||||
if [ -f "${OUTPUT_DIR}/heap_before.pb.gz" ] && [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
|
||||
go tool pprof -top -base="${OUTPUT_DIR}/heap_before.pb.gz" \
|
||||
"${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/heap_growth.txt" 2>&1 || true
|
||||
fi
|
||||
|
||||
# FFI/CGO related
|
||||
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
|
||||
grep -E "(block_on|CGO|FFI|json|Marshal|Unmarshal)" > "${OUTPUT_DIR}/ffi_related.txt" || \
|
||||
echo "No FFI/CGO related functions found" > "${OUTPUT_DIR}/ffi_related.txt"
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo -e "${GREEN}=== Analysis Summary ===${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}CPU Top (Cumulative) - Top 10:${NC}"
|
||||
head -12 "${OUTPUT_DIR}/cpu_top_cum.txt" | tail -10 || true
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}CPU Top (Flat) - Top 10:${NC}"
|
||||
head -12 "${OUTPUT_DIR}/cpu_top_flat.txt" | tail -10 || true
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}FFI/CGO Related Functions:${NC}"
|
||||
cat "${OUTPUT_DIR}/ffi_related.txt" || true
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}=== Detailed Reports ===${NC}"
|
||||
echo "CPU (cumulative): cat ${OUTPUT_DIR}/cpu_top_cum.txt"
|
||||
echo "CPU (flat): cat ${OUTPUT_DIR}/cpu_top_flat.txt"
|
||||
echo "Memory growth: cat ${OUTPUT_DIR}/heap_growth.txt"
|
||||
echo "FFI related: cat ${OUTPUT_DIR}/ffi_related.txt"
|
||||
echo ""
|
||||
echo -e "${GREEN}=== Interactive Analysis ===${NC}"
|
||||
echo "Run: go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz"
|
||||
echo "Then visit: http://localhost:8081/ui/flamegraph"
|
||||
echo ""
|
||||
echo "Profile files saved to: ${OUTPUT_DIR}"
|
||||
37
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/service/sglang.go
vendored
Normal file
37
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/service/sglang.go
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
sglang "github.com/sglang/sglang-go-grpc-sdk"
|
||||
)
|
||||
|
||||
// SGLangService wraps SGLang client
|
||||
type SGLangService struct {
|
||||
client *sglang.Client
|
||||
}
|
||||
|
||||
func NewSGLangService(endpoint, tokenizerPath string) (*SGLangService, error) {
|
||||
client, err := sglang.NewClient(sglang.ClientConfig{
|
||||
Endpoint: endpoint,
|
||||
TokenizerPath: tokenizerPath,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &SGLangService{
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Client returns the underlying SGLang client
|
||||
func (s *SGLangService) Client() *sglang.Client {
|
||||
return s.client
|
||||
}
|
||||
|
||||
// Close closes the SGLang client
|
||||
func (s *SGLangService) Close() error {
|
||||
if s.client != nil {
|
||||
return s.client.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
34
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/utils/utils.go
vendored
Normal file
34
third_party/sglang/sgl-model-gateway/bindings/golang/examples/oai_server/utils/utils.go
vendored
Normal file
@@ -0,0 +1,34 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
// RespondError sends an error response in OpenAI format
|
||||
func RespondError(ctx *fasthttp.RequestCtx, statusCode int, message, errorType string) {
|
||||
ctx.SetStatusCode(statusCode)
|
||||
ctx.SetContentType("application/json")
|
||||
|
||||
response := map[string]interface{}{
|
||||
"error": map[string]interface{}{
|
||||
"message": message,
|
||||
"type": errorType,
|
||||
"code": statusCode,
|
||||
},
|
||||
}
|
||||
|
||||
jsonData, _ := json.Marshal(response)
|
||||
ctx.Write(jsonData)
|
||||
}
|
||||
|
||||
// BuildResponseBase builds the base response structure for OpenAI-compatible responses
|
||||
func BuildResponseBase(id string, created int64, model string) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"id": id,
|
||||
"object": "chat.completion",
|
||||
"created": created,
|
||||
"model": model,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user