chore: vendor sglang v0.5.10 snapshot

This commit is contained in:
2026-04-24 12:29:36 +00:00
parent 78f0d15221
commit bded08301f
4308 changed files with 1200894 additions and 2 deletions

View File

@@ -0,0 +1,239 @@
# Makefile for OAI Server
# Builds binary, runs tests, and provides basic targets
# Configuration
APP_NAME = oai_server
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
BUILD_TIME := $(shell date -u '+%Y-%m-%d_%H:%M:%S')
GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
# Paths
ROOT_DIR := $(shell pwd)
BINDINGS_DIR := $(shell cd $(ROOT_DIR)/../.. && pwd)
BUILD_DIR := $(ROOT_DIR)/build
BINARY := $(BUILD_DIR)/$(APP_NAME)
# Rust FFI library paths
LIB_DIR := $(BINDINGS_DIR)/lib
LIB_NAME = libsgl_model_gateway_go
# Detect OS
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S),Linux)
LIB_EXT = .so
LD_LIBRARY_PATH_VAR = LD_LIBRARY_PATH
ARCH := $(shell uname -m)
ifeq ($(ARCH),x86_64)
GOARCH = amd64
else ifeq ($(ARCH),aarch64)
GOARCH = arm64
endif
endif
ifeq ($(UNAME_S),Darwin)
LIB_EXT = .dylib
LD_LIBRARY_PATH_VAR = DYLD_LIBRARY_PATH
ARCH := $(shell uname -m)
ifeq ($(ARCH),x86_64)
GOARCH = amd64
else ifeq ($(ARCH),arm64)
GOARCH = arm64
endif
endif
# Build flags
LDFLAGS = -X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME) -X main.GitCommit=$(GIT_COMMIT)
GO_BUILD_FLAGS = -ldflags "$(LDFLAGS)"
# Python LDFLAGS (needed for Rust FFI that depends on Python)
PYTHON_LDFLAGS := $(shell python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || python-config --ldflags --embed 2>/dev/null || python-config --ldflags 2>/dev/null || echo "")
# CGO flags
CGO_LDFLAGS = -L$(LIB_DIR) $(PYTHON_LDFLAGS)
.PHONY: all build build-dev test e2e clean help lib run stream check-rust-lib check-server
# E2E test configuration
E2E_HOST ?= localhost
E2E_PORT ?= 8080
E2E_MODEL ?= default
E2E_TOKENIZER ?= $(shell echo $$SGL_TOKENIZER_PATH || echo "./examples/tokenizer")
E2E_NUM_PROMPTS ?= 100
E2E_INPUT_LEN ?= 1024
E2E_OUTPUT_LEN ?= 512
E2E_REQUEST_RATE ?= 20
E2E_MAX_CONCURRENCY ?= 20
E2E_BASE_URL ?= http://$(E2E_HOST):$(E2E_PORT)
help:
@echo "OAI Server Makefile"
@echo ""
@echo "Available targets:"
@echo " lib - Build Rust FFI library"
@echo " build - Build binary (release mode)"
@echo " build-dev - Build binary (debug mode)"
@echo " test - Run tests"
@echo " e2e - Run end-to-end test with bench_serving.py"
@echo " run - Run the server (development)"
@echo " stream - Run streaming example"
@echo " clean - Clean build artifacts"
@echo ""
@echo "E2E test variables:"
@echo " E2E_HOST - OAI Server host (default: localhost)"
@echo " E2E_PORT - OAI Server port (default: 8080)"
@echo " E2E_MODEL - Model name (default: default)"
@echo " E2E_TOKENIZER - Tokenizer path"
@echo " E2E_NUM_PROMPTS - Number of prompts (default: 100)"
@echo " E2E_INPUT_LEN - Input token length (default: 1024)"
@echo " E2E_OUTPUT_LEN - Output token length (default: 512)"
@echo " E2E_REQUEST_RATE - Request rate per second (default: 20)"
@echo " E2E_MAX_CONCURRENCY - Max concurrent requests (default: 20)"
all: build
# Build Rust FFI library
lib:
@echo "Building Rust FFI library..."
@cd $(BINDINGS_DIR) && $(MAKE) lib
@echo "✓ Rust FFI library built"
# Check if Rust FFI library exists
check-rust-lib:
@if [ ! -f "$(LIB_DIR)/$(LIB_NAME)$(LIB_EXT)" ]; then \
echo "Error: Rust FFI library not found at $(LIB_DIR)/$(LIB_NAME)$(LIB_EXT)"; \
echo "Building Rust library..."; \
cd $(BINDINGS_DIR) && $(MAKE) lib; \
fi
@echo "✓ Rust FFI library found"
# Build binary (release)
build: check-rust-lib
@echo "Building $(APP_NAME) (release mode)..."
@mkdir -p $(BUILD_DIR)
@CGO_ENABLED=1 \
CGO_LDFLAGS="$(CGO_LDFLAGS)" \
GOOS=$(shell go env GOOS) \
GOARCH=$(GOARCH) \
go build $(GO_BUILD_FLAGS) -o $(BINARY) .
@echo "✓ Binary built: $(BINARY)"
# Build binary (debug)
build-dev: check-rust-lib
@echo "Building $(APP_NAME) (debug mode)..."
@mkdir -p $(BUILD_DIR)
@CGO_ENABLED=1 \
CGO_LDFLAGS="$(CGO_LDFLAGS)" \
go build -o $(BINARY) .
@echo "✓ Binary built (debug): $(BINARY)"
# Run tests
test: check-rust-lib
@echo "Running tests..."
@CGO_ENABLED=1 \
CGO_LDFLAGS="$(CGO_LDFLAGS)" \
export $(LD_LIBRARY_PATH_VAR)="$(LIB_DIR):$$$(LD_LIBRARY_PATH_VAR)" && \
go test -v ./...
@echo "✓ Tests completed"
# Check if OAI Server is running
check-server:
@echo "Checking if OAI Server is running at $(E2E_BASE_URL)..."
@if curl -s -f $(E2E_BASE_URL)/health > /dev/null 2>&1; then \
echo "✓ OAI Server is running"; \
exit 0; \
else \
echo "✗ OAI Server is not running at $(E2E_BASE_URL)"; \
echo " Start it with: make run"; \
exit 1; \
fi
# Find sglang project root (4 levels up from oai_server)
SGLANG_ROOT := $(shell cd $(ROOT_DIR)/../../../../.. && pwd)
# Run end-to-end test with bench_serving.py
e2e: check-server
@echo "Checking if bench_serving.py is available..."
@if python -m sglang.bench_serving --help > /dev/null 2>&1; then \
echo "✓ Using installed bench_serving.py module"; \
USE_SGLANG_ROOT=false; \
elif [ -f "$(SGLANG_ROOT)/python/sglang/bench_serving.py" ]; then \
echo "✓ Using bench_serving.py from $(SGLANG_ROOT)"; \
USE_SGLANG_ROOT=true; \
else \
echo "✗ bench_serving.py is not available"; \
echo " Install dependencies: pip install aiohttp numpy datasets transformers tqdm pillow pybase64"; \
exit 1; \
fi
@echo "Running end-to-end test with bench_serving.py..."
@echo "Configuration:"
@echo " Server: $(E2E_BASE_URL)"
@if [ "$(E2E_MODEL)" != "default" ]; then \
echo " Model: $(E2E_MODEL)"; \
fi
@if [ -n "$(E2E_TOKENIZER)" ]; then \
echo " Tokenizer: $(E2E_TOKENIZER)"; \
fi
@echo " Prompts: $(E2E_NUM_PROMPTS)"
@echo " Input/Output: $(E2E_INPUT_LEN)/$(E2E_OUTPUT_LEN) tokens"
@echo " Request rate: $(E2E_REQUEST_RATE) req/s"
@echo " Max concurrency: $(E2E_MAX_CONCURRENCY)"
@echo ""
@TOKENIZER_ABS=$$(cd $(ROOT_DIR) && python3 -c "import os; path='$(E2E_TOKENIZER)'; print(os.path.abspath(path) if not os.path.isabs(path) else path)" 2>/dev/null || echo "$(E2E_TOKENIZER)"); \
if [ -n "$(E2E_TOKENIZER)" ]; then \
if [ -n "$$TOKENIZER_ABS" ] && ([ -d "$$TOKENIZER_ABS" ] || [ -f "$$TOKENIZER_ABS" ]); then \
TOKENIZER_ARG="--tokenizer $$TOKENIZER_ABS"; \
else \
TOKENIZER_ARG="--tokenizer $(E2E_TOKENIZER)"; \
fi; \
else \
TOKENIZER_ARG=""; \
fi; \
if [ "$$USE_SGLANG_ROOT" = "true" ]; then \
cd $(SGLANG_ROOT) && PYTHONPATH=$(SGLANG_ROOT)/python:$$PYTHONPATH python python/sglang/bench_serving.py \
--backend sglang-oai-chat \
--base-url $(E2E_BASE_URL) \
$$([ "$(E2E_MODEL)" != "default" ] && echo "--model $(E2E_MODEL)") \
$$TOKENIZER_ARG \
--dataset-name random \
--num-prompts $(E2E_NUM_PROMPTS) \
--random-input-len $(E2E_INPUT_LEN) \
--random-output-len $(E2E_OUTPUT_LEN) \
--request-rate $(E2E_REQUEST_RATE) \
--max-concurrency $(E2E_MAX_CONCURRENCY) \
--warmup-requests 5 \
--disable-tqdm || (echo "✗ E2E test failed"; exit 1); \
else \
python -m sglang.bench_serving \
--backend sglang-oai-chat \
--base-url $(E2E_BASE_URL) \
$$([ "$(E2E_MODEL)" != "default" ] && echo "--model $(E2E_MODEL)") \
$$TOKENIZER_ARG \
--dataset-name random \
--num-prompts $(E2E_NUM_PROMPTS) \
--random-input-len $(E2E_INPUT_LEN) \
--random-output-len $(E2E_OUTPUT_LEN) \
--request-rate $(E2E_REQUEST_RATE) \
--max-concurrency $(E2E_MAX_CONCURRENCY) \
--warmup-requests 5 \
--disable-tqdm || (echo "✗ E2E test failed"; exit 1); \
fi
@echo ""
@echo "✓ E2E test completed"
# Run the server (development)
run: build-dev
@echo "Running server..."
@export $(LD_LIBRARY_PATH_VAR)="$(LIB_DIR):$$$(LD_LIBRARY_PATH_VAR)" && \
$(BINARY)
# Run streaming example
stream: check-rust-lib
@echo "Running streaming example..."
@cd $(BINDINGS_DIR)/examples/streaming && \
export $(LD_LIBRARY_PATH_VAR)="$(LIB_DIR):$$$(LD_LIBRARY_PATH_VAR)" && \
bash run.sh
# Clean build artifacts
clean:
@echo "Cleaning build artifacts..."
@rm -rf $(BUILD_DIR)
@echo "✓ Clean complete"

View File

@@ -0,0 +1,305 @@
# Go SGLang Router - OpenAI Compatible API Server
Go SGLang Router is a high-performance OpenAI-compatible API server that communicates with the SGLang backend via gRPC and performs efficient preprocessing and postprocessing through Rust FFI.
## Features
-**OpenAI API Compatible**: Fully compatible with OpenAI Chat Completions API
-**High Performance**: Low latency and high throughput using gRPC and Rust FFI
-**Streaming Support**: Server-Sent Events (SSE) streaming responses
-**Thread-Safe**: Pre-created tokenizer handle, lock-free concurrency
-**Graceful Shutdown**: Context cancellation mechanism to avoid resource leaks and panics
-**Configurable**: Supports configuring channel buffer sizes and timeout durations
## Architecture Overview
**Important Note**: gRPC mode **still calls FFI**, which is used for:
- **Preprocessing**: chat_template and tokenization (request phase)
- **Postprocessing**: token decoding and tool parsing (response phase)
gRPC is only used for communication with the SGLang backend, while input/output processing completely relies on Rust FFI.
```
┌─────────────────────────────────────────────────────────────────┐
│ HTTP Client │
│ (OpenAI API Format) │
└────────────────────────────┬────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ FastHTTP Server │
│ handlers/chat.go:HandleChatCompletion │
│ - Parse request JSON │
│ - SetBodyStreamWriter (SSE) │
└────────────────────────────┬────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ SGLang Client (client.go) │
│ CreateChatCompletionStream(ctx, req) │
│ - Wraps gRPC client │
└────────────────────────────┬────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ gRPC Client (internal/grpc/client_grpc.go) │
│ CreateChatCompletionStream(ctx, reqJSON) │
│ │
│ ┌──────────────────────────────────────────────────────────┐ │
│ │ Step 1: FFI Preprocess (Rust FFI) │ │
│ │ - ffi.PreprocessChatRequestWithTokenizer() │ │
│ │ - chat_template application │ │
│ │ - tokenization │ │
│ │ - tool constraints generation │ │
│ │ Returns: PromptText, TokenIDs, ToolConstraintsJSON, │ │
│ │ PromptTokens │ │
│ └────────────────────┬─────────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌──────────────────────────────────────────────────────────┐ │
│ │ Step 2: Build gRPC Request │ │
│ │ - Parse request JSON (model, temperature, etc.) │ │
│ │ - Create proto.GenerateRequest │ │
│ │ - Set TokenizedInput (PromptText, TokenIDs) │ │
│ │ - Set SamplingParams (temperature, top_p, top_k, etc.) │ │
│ │ - Set Constraints (from ToolConstraintsJSON) │ │
│ └────────────────────┬─────────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌──────────────────────────────────────────────────────────┐ │
│ │ Step 3: Create gRPC Stream │ │
│ │ - client.Generate(generateReq) → gRPC stream │ │
│ │ - Connects to SGLang Backend (Rust) │ │
│ └────────────────────┬─────────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌──────────────────────────────────────────────────────────┐ │
│ │ Step 4: Create Converter & BatchPostprocessor │ │
│ │ - ffi.CreateGrpcResponseConverterWithTokenizer() │ │
│ │ - Uses preprocessed.PromptTokens for initial count │ │
│ │ - ffi.NewBatchPostprocessor(batchSize=1, immediate) │ │
│ └────────────────────┬─────────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌──────────────────────────────────────────────────────────┐ │
│ │ Step 5: Start readLoop (Background Goroutine) │ │
│ │ - go grpcStream.readLoop() │ │
│ │ - Returns GrpcChatCompletionStream immediately │ │
│ └────────────────────┬─────────────────────────────────────┘ │
└───────────────────────┼────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ GrpcChatCompletionStream.readLoop() │
│ (Background Goroutine) │
│ │
│ ┌──────────────────────────────────────────────────────────┐ │
│ │ Recv() Goroutine (Dedicated) │ │
│ │ - Continuously calls stream.Recv() │ │
│ │ - Sends results to recvChan (buffered, 2000) │ │
│ │ - Exits on ctx.Done() or error │ │
│ │ - Calls stream.CloseSend() on ctx.Done() │ │
│ └────────────────────┬─────────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌──────────────────────────────────────────────────────────┐ │
│ │ Main Loop │ │
│ │ - Reads from recvChan │ │
│ │ - For each proto.GenerateResponse: │ │
│ │ → go processAndSendResponse() (async) │ │
│ │ - protoToJSON() converts proto to JSON string │ │
│ │ - batchPostprocessor.AddChunk(protoJSON) │ │
│ │ → FFI postprocessing (token decoding, tool parsing)│ │
│ │ → Returns OpenAI-format JSON strings │ │
│ │ - Sends JSON to resultJSONChan (buffered, 10000) │ │
│ │ - All operations check ctx.Done() for cancellation │ │
│ │ - On EOF: flush batch, send remaining results, return │ │
│ │ - On error: send to errChan (buffered, 100) │ │
│ │ - defer: cancel ctx, wait goroutines, close channels │ │
│ └────────────────────┬─────────────────────────────────────┘ │
└───────────────────────┼────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ resultJSONChan (Buffered Channel, 10000) │
│ - Contains OpenAI-format JSON strings │
│ - Ready for consumption │
└────────────────────────────┬────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ ChatCompletionStream.RecvJSON() │
│ (client.go:410) │
│ - Direct wrapper: return grpcStream.RecvJSON() │
│ - No intermediate processing │
└────────────────────────────┬────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ FastHTTP SetBodyStreamWriter │
│ (handlers/chat.go:159) │
│ - Loop: stream.RecvJSON() → format SSE → flush │
│ - Format: "data: {json}\n\n" │
│ - Final: "data: [DONE]\n\n" │
│ - Immediate flush after each chunk │
└────────────────────────────┬────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ HTTP Client │
│ (SSE Stream) │
│ Receives: data: {...}\n\n │
└─────────────────────────────────────────────────────────────────┘
```
## Quick Start
### Start Server
```bash
./run.sh
```
The server will start on port `:8080`.
### Usage Example
```bash
curl http://localhost:8080/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "/path/to/model",
"messages": [{"role": "user", "content": "Hello!"}],
"stream": true
}'
```
## Key Design
### 1. Thread-Safe Tokenizer
- Pre-create `TokenizerHandle` at startup
- Rust side uses `Arc<dyn TokenizerTrait>`, thread-safe
- Lock-free concurrency, eliminating lock contention
### 2. Context Cancellation Mechanism (Graceful Shutdown)
- Use `context.Context` cancellation mechanism
- In `readLoop`'s `defer`: cancel context first, then wait for all goroutines to complete, finally close channels
- `processAndSendResponse` checks `ctx.Done()` at function start, all `select` statements include `case <-s.ctx.Done()`
- Avoids "send on closed channel" panic
### 3. Cancellable Recv()
- Use dedicated goroutine to execute `Recv()`
- Pass results through `recvChan`
- Call `CloseSend()` when context is cancelled to make `Recv()` return error
### 4. Simplified Channel Design
- `resultJSONChan`: Main data channel (gRPC layer)
- `errChan`: Error channel (gRPC layer)
- `recvChan`: Internal communication channel (gRPC layer)
- Removed redundant channels and duplicate reads
## Configuration
### Channel Buffer Sizes
```go
type ChannelBufferSizes struct {
ResultJSONChan int // Default: 10000
ErrChan int // Default: 100
RecvChan int // Default: 2000
}
```
### Timeout Configuration
```go
type Timeouts struct {
KeepaliveTime time.Duration // Default: 300s
KeepaliveTimeout time.Duration // Default: 20s
CloseTimeout time.Duration // Default: 5s
}
```
## Performance Optimizations
1. **Pre-create Tokenizer**: Created at startup to avoid first request latency
2. **Lock-Free Concurrency**: Tokenizer is thread-safe, no locks needed
3. **Lazy Parsing**: JSON parsing deferred until needed
4. **Direct JSON Passing**: `RecvJSON()` avoids parse/serialize overhead
5. **Immediate Batching**: batchSize=1, no delay
6. **Async Processing**: `readLoop` processes in background, doesn't block request handling
7. **Configurable Buffers**: Adjust channel sizes based on concurrency needs
## File Structure
```
sgl-model-gateway/bindings/golang/
├── client.go # High-level client API
├── internal/
│ ├── grpc/
│ │ └── client_grpc.go # gRPC client implementation
│ ├── ffi/ # FFI bindings (Rust)
│ └── proto/ # Protobuf definitions
└── examples/
└── oai_server/
├── handlers/
│ └── chat.go # HTTP request handling
├── models/
│ └── chat.go # Request/response models
└── service/
└── sglang_service.go # Service layer
```
## Error Handling
### Context Cancellation Mechanism
1. **Client disconnects**`SetBodyStreamWriter` detects flush error
2. **Cancel streamCtx**`readLoop` detects `ctx.Done()`
3. **Call stream.CloseSend()**`Recv()` goroutine returns error
4. **readLoop defer executes**:
- Set `closed` flag
- Cancel context (if not already cancelled)
- Wait for all `processAndSendResponse` goroutines to complete (`processWg.Wait()`)
- Close all channels (`resultJSONChan`, `errChan`, `readLoopDone`)
5. **Clean up resources and exit**
### Channel Blocking and Race Condition Prevention
- **Context cancellation mechanism**: All channel sends use `select` statements with `case <-s.ctx.Done()`
- **Graceful exit**: When context is cancelled, all blocking send operations can return immediately
- **WaitGroup synchronization**: `readLoop`'s `defer` uses `processWg.Wait()` to ensure all goroutines complete before closing channels
- **Avoid panic**: Through context cancellation and WaitGroup synchronization, avoids "send on closed channel" panic
## Key Functions
### CreateChatCompletionStream
**Location**: `internal/grpc/client_grpc.go:108`
- Preprocess request (FFI)
- Build gRPC request
- Create converter and batch processor
- Start `readLoop`
### readLoop
**Location**: `internal/grpc/client_grpc.go:290`
- Start Recv() goroutine (continuously calls `stream.Recv()`)
- Process proto responses
- Asynchronously call `processAndSendResponse` (tracked with `processWg`)
- **Graceful shutdown in defer**:
- Set `closed` flag
- Cancel context (if not already cancelled)
- Wait for all `processAndSendResponse` goroutines to complete (`processWg.Wait()`)
- Close all channels (`resultJSONChan`, `errChan`, `readLoopDone`)
### processAndSendResponse
**Location**: `internal/grpc/client_grpc.go:379`
- Check `ctx.Done()` at function start, return immediately if cancelled
- Convert proto to JSON
- Call FFI batch processor
- All `select` statements include `case <-s.ctx.Done()` for graceful shutdown handling
- Send JSON to channel
### RecvJSON
**Location**:
- `internal/grpc/client_grpc.go:412`: gRPC layer implementation
- `client.go:410`: Client wrapper layer
- Read from `resultJSONChan`
- Directly return JSON string, no parsing needed

View File

@@ -0,0 +1,55 @@
package config
import (
"os"
)
// Config holds the application configuration
type Config struct {
Endpoint string
TokenizerPath string
Port string
LogDir string
LogLevel string
}
// Load loads configuration from environment variables with defaults
func Load() *Config {
// Get tokenizer path from environment or use default
tokenizerPath := os.Getenv("SGL_TOKENIZER_PATH")
if tokenizerPath == "" {
tokenizerPath = "../tokenizer"
}
// Get endpoint from environment or use default
endpoint := os.Getenv("SGL_GRPC_ENDPOINT")
if endpoint == "" {
endpoint = "grpc://localhost:20000"
}
// Get port from environment or use default
port := os.Getenv("PORT")
if port == "" {
port = "8080"
}
// Get log directory from environment or use default
logDir := os.Getenv("LOG_DIR")
if logDir == "" {
logDir = "./logs"
}
// Get log level from environment or use default
logLevel := os.Getenv("LOG_LEVEL")
if logLevel == "" {
logLevel = "info"
}
return &Config{
Endpoint: endpoint,
TokenizerPath: tokenizerPath,
Port: port,
LogDir: logDir,
LogLevel: logLevel,
}
}

View File

@@ -0,0 +1,121 @@
/tmp/ShareGPT_V3_unfiltered_cleaned_split.json: 100%|████████████████████| 642M/642M [10:02<00:00, 1.12MB/s]
#Input tokens: 50561
#Output tokens: 25883
Starting warmup with 5 sequences...
Warmup completed with 5 sequences. Starting main benchmark run...
============ Serving Benchmark Result ============
Backend: sglang-oai-chat
Traffic request rate: 20.0
Max request concurrency: 20
Successful requests: 100
Benchmark duration (s): 107.24
Total input tokens: 50561
Total input text tokens: 50561
Total input vision tokens: 0
Total generated tokens: 25883
Total generated tokens (retokenized): 129591
Request throughput (req/s): 0.93
Input token throughput (tok/s): 471.48
Output token throughput (tok/s): 241.36
Total token throughput (tok/s): 712.84
Concurrency: 16.42
----------------End-to-End Latency----------------
Mean E2E Latency (ms): 17609.46
Median E2E Latency (ms): 12343.82
---------------Time to First Token----------------
Mean TTFT (ms): 190.71
Median TTFT (ms): 164.86
P99 TTFT (ms): 397.72
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms): 162.55
Median TPOT (ms): 63.51
P99 TPOT (ms): 1337.20
---------------Inter-Token Latency----------------
Mean ITL (ms): 25.85
Median ITL (ms): 24.26
P95 ITL (ms): 48.26
P99 ITL (ms): 119.04
Max ITL (ms): 194.58
==================================================
E2E test completed
## Rust
============ Serving Benchmark Result ============
Backend: sglang-oai-chat
Traffic request rate: 20.0
Max request concurrency: 20
Successful requests: 100
Benchmark duration (s): 37.71
Total input tokens: 50561
Total input text tokens: 50561
Total input vision tokens: 0
Total generated tokens: 25883
Total generated tokens (retokenized): 25599
Request throughput (req/s): 2.65
Input token throughput (tok/s): 1340.75
Output token throughput (tok/s): 686.35
Total token throughput (tok/s): 2027.10
Concurrency: 18.58
----------------End-to-End Latency----------------
Mean E2E Latency (ms): 7008.05
Median E2E Latency (ms): 7061.24
---------------Time to First Token----------------
Mean TTFT (ms): 156.09
Median TTFT (ms): 133.81
P99 TTFT (ms): 318.53
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms): 26.59
Median TPOT (ms): 26.75
P99 TPOT (ms): 29.18
---------------Inter-Token Latency----------------
Mean ITL (ms): 26.71
Median ITL (ms): 23.61
P95 ITL (ms): 66.11
P99 ITL (ms): 115.30
Max ITL (ms): 201.08
==================================================
## golang
#Input tokens: 50561
#Output tokens: 25883
Starting warmup with 5 sequences...
Warmup completed with 5 sequences. Starting main benchmark run...
============ Serving Benchmark Result ============
Backend: sglang-oai-chat
Traffic request rate: 20.0
Max request concurrency: 20
Successful requests: 100
Benchmark duration (s): 34.22
Total input tokens: 50561
Total input text tokens: 50561
Total input vision tokens: 0
Total generated tokens: 22970
Total generated tokens (retokenized): 31740
Request throughput (req/s): 2.92
Input token throughput (tok/s): 1477.70
Output token throughput (tok/s): 671.32
Total token throughput (tok/s): 2149.03
Concurrency: 18.42
----------------End-to-End Latency----------------
Mean E2E Latency (ms): 6303.33
Median E2E Latency (ms): 6294.46
---------------Time to First Token----------------
Mean TTFT (ms): 157.10
Median TTFT (ms): 149.16
P99 TTFT (ms): 251.98
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms): 26.49
Median TPOT (ms): 27.15
P99 TPOT (ms): 28.73
---------------Inter-Token Latency----------------
Mean ITL (ms): 26.97
Median ITL (ms): 24.61
P95 ITL (ms): 52.39
P99 ITL (ms): 86.52
Max ITL (ms): 194.55
==================================================

View File

@@ -0,0 +1,60 @@
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo=
golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,556 @@
package handlers
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"strings"
"time"
sglang "github.com/sglang/sglang-go-grpc-sdk"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
"oai_server/models"
"oai_server/service"
"oai_server/utils"
)
// ChatHandler handles chat completion requests
type ChatHandler struct {
logger *zap.Logger
service *service.SGLangService
}
// NewChatHandler creates a new chat handler
func NewChatHandler(logger *zap.Logger, svc *service.SGLangService) *ChatHandler {
return &ChatHandler{
logger: logger,
service: svc,
}
}
// recvResult holds the result of a RecvJSON() call
type recvResult struct {
chunkJSON string
err error
}
// HandleChatCompletion handles POST /v1/chat/completions
func (h *ChatHandler) HandleChatCompletion(ctx *fasthttp.RequestCtx) {
var req models.ChatRequest
if err := json.Unmarshal(ctx.PostBody(), &req); err != nil {
h.logger.Warn("Invalid chat completion request", zap.Error(err))
utils.RespondError(ctx, 400, fmt.Sprintf("Invalid request: %v", err), "invalid_request_error")
return
}
path := string(ctx.Path())
defer func() {
statusCode := ctx.Response.StatusCode()
if statusCode == 0 {
statusCode = 200
}
h.logHTTPResponse(statusCode, path)
}()
// Convert to SGLang format
messages := make([]sglang.ChatMessage, len(req.Messages))
for i, msg := range req.Messages {
role, roleOk := msg["role"]
content, contentOk := msg["content"]
// Validate role
if !roleOk || role == "" {
h.logger.Warn("Missing or empty role in message", zap.Int("message_index", i))
utils.RespondError(ctx, 400, "Message role is required and cannot be empty", "invalid_request_error")
return
}
// Ensure content is always a string (not null)
// Chat template requires content field to be present, even if empty
// If content is missing or null, use empty string
contentStr := ""
if contentOk && content != "" {
contentStr = content
}
messages[i] = sglang.ChatMessage{
Role: role,
Content: contentStr,
}
}
sglReq := sglang.ChatCompletionRequest{
Model: req.Model,
Messages: messages,
Stream: req.Stream,
}
if req.Temperature != nil {
temp := float32(*req.Temperature)
sglReq.Temperature = &temp
}
if req.TopP != nil {
topP := float32(*req.TopP)
sglReq.TopP = &topP
}
if req.MaxCompletionTokens != nil {
sglReq.MaxCompletionTokens = req.MaxCompletionTokens
} else if req.MaxTokens != nil {
sglReq.MaxCompletionTokens = req.MaxTokens
}
requestCtx := context.Background()
if req.Stream {
h.handleStreamingCompletion(ctx, requestCtx, sglReq)
} else {
h.handleNonStreamingCompletion(ctx, requestCtx, sglReq)
}
}
// isBrokenPipeError checks if the error is a broken pipe error (client disconnected)
func isBrokenPipeError(err error) bool {
if err == nil {
return false
}
errStr := err.Error()
return strings.Contains(errStr, "broken pipe") ||
strings.Contains(errStr, "connection reset by peer") ||
strings.Contains(errStr, "connection closed") ||
strings.Contains(errStr, "write: connection closed")
}
// logHTTPResponse logs HTTP response with colored output
func (h *ChatHandler) logHTTPResponse(statusCode int, path string) {
var statusText string
var colorCode string
switch {
case statusCode >= 200 && statusCode < 300:
colorCode = "\033[32m" // Green
statusText = "OK"
case statusCode >= 300 && statusCode < 400:
colorCode = "\033[33m" // Yellow
statusText = "Redirect"
case statusCode >= 400 && statusCode < 500:
colorCode = "\033[33m" // Yellow
statusText = "Client Error"
case statusCode >= 500:
colorCode = "\033[31m" // Red
statusText = "Server Error"
default:
colorCode = "\033[37m" // White
statusText = "Unknown"
}
resetCode := "\033[0m"
msg := fmt.Sprintf("%s[%d %s]%s %s", colorCode, statusCode, statusText, resetCode, path)
h.logger.Info(msg)
}
func (h *ChatHandler) handleStreamingCompletion(ctx *fasthttp.RequestCtx, requestCtx context.Context, req sglang.ChatCompletionRequest) {
ctx.SetContentType("text/event-stream")
ctx.Response.Header.Set("Cache-Control", "no-cache")
ctx.Response.Header.Set("Connection", "keep-alive")
ctx.Response.Header.Set("X-Accel-Buffering", "no")
ctx.SetStatusCode(200)
var clientDisconnected bool
// Flush timeout: prevent deadlock if client is slow or disconnected
// This timeout should be longer than typical network latency but shorter than client timeout
const flushTimeout = 5 * time.Second
ctx.SetBodyStreamWriter(func(w *bufio.Writer) {
streamCtx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := h.service.Client().CreateChatCompletionStream(streamCtx, req)
if err != nil {
h.logger.Error("Failed to create chat completion stream",
zap.Error(err),
zap.String("model", req.Model),
)
// Use sendSSEError to send error in consistent format
errInfo, sendErr := h.sendSSEError(w, err)
if sendErr != nil {
h.logger.Warn("Failed to send SSE error", zap.Error(sendErr))
} else if errInfo.IsTimeout {
h.logger.Error("Stream creation timeout", zap.Error(err))
}
return
}
defer func() {
if closeErr := stream.Close(); closeErr != nil {
h.logger.Warn("Failed to close stream", zap.Error(closeErr))
}
}()
// Use a single dedicated goroutine to continuously call RecvJSON() and send results via channel
recvChan := make(chan recvResult, 20)
recvGoroutineDone := make(chan struct{})
go func() {
defer func() {
close(recvChan)
close(recvGoroutineDone)
}()
for {
// Check context before calling RecvJSON() to avoid blocking if context is cancelled
select {
case <-streamCtx.Done():
return
default:
}
// Call RecvJSON() - this may block, but stream.Close() will unblock it
// when context is cancelled (called from main loop)
chunkJSON, err := stream.RecvJSON()
// Check context again after RecvJSON() returns
select {
case <-streamCtx.Done():
return
default:
}
// Send to channel (may block if channel is full)
// If channel is full, this will block until main loop reads from it
// This is acceptable because main loop should be actively reading
select {
case recvChan <- recvResult{chunkJSON: chunkJSON, err: err}:
if err != nil {
// EOF or other error, stop the goroutine
return
}
case <-streamCtx.Done():
// Context cancelled while sending, stop the goroutine
return
}
}
}()
for {
if clientDisconnected {
cancel()
// Close stream immediately to unblock RecvJSON() calls
stream.Close()
return
}
select {
case <-streamCtx.Done():
// Close stream to ensure RecvJSON() goroutine can exit
stream.Close()
return
case result, ok := <-recvChan:
if !ok {
// Channel closed, stream ended
return
}
if result.err == io.EOF {
if !clientDisconnected {
w.WriteString("data: [DONE]\n\n")
// Flush with timeout to prevent deadlock
flushDone := make(chan error, 1)
go func() {
flushDone <- w.Flush()
}()
flushCtx, flushCancel := context.WithTimeout(streamCtx, flushTimeout)
defer flushCancel()
select {
case flushErr := <-flushDone:
if flushErr != nil && !isBrokenPipeError(flushErr) {
h.logger.Warn("Final flush error", zap.Error(flushErr))
}
case <-flushCtx.Done():
if flushCtx.Err() == context.DeadlineExceeded {
h.logger.Warn("Final flush timeout", zap.Duration("timeout", flushTimeout))
}
case <-streamCtx.Done():
// Context cancelled, skip flush
}
}
return
}
if result.err != nil {
if result.err == context.Canceled || result.err == context.DeadlineExceeded {
return
}
// Send error to client before closing
errInfo, sendErr := h.sendSSEError(w, result.err)
if sendErr != nil {
h.logger.Warn("Failed to send SSE error", zap.Error(sendErr))
}
if errInfo.IsTimeout {
h.logger.Error("Stream timeout error", zap.Error(result.err))
} else {
h.logger.Error("Stream error", zap.Error(result.err))
}
return
}
if result.chunkJSON == "" {
continue
}
w.WriteString("data: ")
w.WriteString(result.chunkJSON)
w.WriteString("\n\n")
// Flush with timeout to prevent deadlock:
// If Flush blocks indefinitely (slow client), RecvJSON goroutine may fill recvChan
// and then block trying to send, causing deadlock
// Note: bufio.Writer.Flush() doesn't have a timeout parameter, so we use
// a goroutine + select pattern to implement timeout behavior
flushDone := make(chan error, 1)
go func() {
flushDone <- w.Flush()
}()
flushCtx, flushCancel := context.WithTimeout(streamCtx, flushTimeout)
defer flushCancel()
select {
case err := <-flushDone:
if err != nil {
if isBrokenPipeError(err) {
clientDisconnected = true
cancel()
// Close stream immediately to unblock RecvJSON() calls
stream.Close()
return
}
h.logger.Warn("Flush error", zap.Error(err))
}
case <-flushCtx.Done():
// Flush timeout: client may be slow or disconnected
// Continue processing to avoid deadlock, but mark as disconnected
if flushCtx.Err() == context.DeadlineExceeded {
h.logger.Warn("Flush timeout, client may be slow or disconnected", zap.Duration("timeout", flushTimeout))
}
clientDisconnected = true
cancel()
stream.Close()
return
case <-streamCtx.Done():
// Context cancelled, stop flushing
return
}
}
}
})
}
func (h *ChatHandler) handleNonStreamingCompletion(ctx *fasthttp.RequestCtx, requestCtx context.Context, req sglang.ChatCompletionRequest) {
resp, err := h.service.Client().CreateChatCompletion(requestCtx, req)
if err != nil {
h.logger.Error("Failed to create chat completion",
zap.Error(err),
zap.String("model", req.Model),
)
utils.RespondError(ctx, 500, fmt.Sprintf("Failed to create completion: %v", err), "server_error")
return
}
// Convert to OpenAI format
response := utils.BuildResponseBase(resp.ID, resp.Created, resp.Model)
response["object"] = "chat.completion"
choices := make([]map[string]interface{}, len(resp.Choices))
for i, choice := range resp.Choices {
choiceMap := map[string]interface{}{
"index": choice.Index,
"message": map[string]interface{}{
"role": choice.Message.Role,
"content": choice.Message.Content,
},
"finish_reason": choice.FinishReason,
}
if len(choice.Message.ToolCalls) > 0 {
toolCalls := make([]map[string]interface{}, len(choice.Message.ToolCalls))
for j, tc := range choice.Message.ToolCalls {
toolCalls[j] = map[string]interface{}{
"id": tc.ID,
"type": tc.Type,
"function": map[string]interface{}{"name": tc.Function.Name, "arguments": tc.Function.Arguments},
}
}
choiceMap["message"].(map[string]interface{})["tool_calls"] = toolCalls
}
choices[i] = choiceMap
}
response["choices"] = choices
// Usage is always present (not a pointer)
response["usage"] = map[string]interface{}{
"prompt_tokens": resp.Usage.PromptTokens,
"completion_tokens": resp.Usage.CompletionTokens,
"total_tokens": resp.Usage.TotalTokens,
}
ctx.SetStatusCode(200)
ctx.SetContentType("application/json")
jsonData, _ := json.Marshal(response)
ctx.Write(jsonData)
}
// StreamErrorInfo holds parsed error information
type StreamErrorInfo struct {
Message string
Type string
Code int
IsTimeout bool
}
// parseStreamError parses error type and code
func parseStreamError(err error) StreamErrorInfo {
if err == nil {
return StreamErrorInfo{}
}
errorMsg := err.Error()
// Check timeout error by message prefix
isTimeout := strings.HasPrefix(errorMsg, "stream.Recv() timeout") || strings.Contains(errorMsg, "timeout after")
errorType := "server_error"
errorCode := 500
if isTimeout {
errorType = "timeout_error"
errorCode = 504
}
return StreamErrorInfo{
Message: errorMsg,
Type: errorType,
Code: errorCode,
IsTimeout: isTimeout,
}
}
// formatErrorJSON formats error as OpenAI JSON
func formatErrorJSON(errInfo StreamErrorInfo) string {
errorObj := map[string]interface{}{
"error": map[string]interface{}{
"message": errInfo.Message,
"type": errInfo.Type,
"code": errInfo.Code,
},
}
jsonBytes, _ := json.Marshal(errorObj)
return string(jsonBytes)
}
// sendSSEError sends SSE error response. Callers should log errors.
func (h *ChatHandler) sendSSEError(w *bufio.Writer, err error) (StreamErrorInfo, error) {
errInfo := parseStreamError(err)
errorJSON := formatErrorJSON(errInfo)
w.WriteString("data: ")
w.WriteString(errorJSON)
w.WriteString("\n\n")
if flushErr := w.Flush(); flushErr != nil && !isBrokenPipeError(flushErr) {
h.logger.Warn("Failed to flush error response", zap.Error(flushErr))
return errInfo, flushErr
}
return errInfo, nil
}
// HandleGenerate handles POST /generate (SGLang native API)
func (h *ChatHandler) HandleGenerate(ctx *fasthttp.RequestCtx) {
path := string(ctx.Path())
defer func() {
statusCode := ctx.Response.StatusCode()
if statusCode == 0 {
statusCode = 200
}
h.logHTTPResponse(statusCode, path)
}()
// Parse request body
var req map[string]interface{}
if err := json.Unmarshal(ctx.PostBody(), &req); err != nil {
h.logger.Warn("Invalid generate request", zap.Error(err))
utils.RespondError(ctx, 400, fmt.Sprintf("Invalid request: %v", err), "invalid_request_error")
return
}
// Extract text and sampling_params
text, ok := req["text"].(string)
if !ok || text == "" {
utils.RespondError(ctx, 400, "Missing or invalid 'text' field", "invalid_request_error")
return
}
samplingParams, _ := req["sampling_params"].(map[string]interface{})
if samplingParams == nil {
samplingParams = make(map[string]interface{})
}
// Convert to chat completion format for processing
chatReq := sglang.ChatCompletionRequest{
Model: "default",
Messages: []sglang.ChatMessage{{Role: "user", Content: text}},
Stream: false,
}
// Copy sampling params
if maxNewTokens, ok := samplingParams["max_new_tokens"].(float64); ok {
tokens := int(maxNewTokens)
chatReq.MaxCompletionTokens = &tokens
}
if temp, ok := samplingParams["temperature"].(float64); ok {
temp32 := float32(temp)
chatReq.Temperature = &temp32
}
if topP, ok := samplingParams["top_p"].(float64); ok {
topP32 := float32(topP)
chatReq.TopP = &topP32
}
if topK, ok := samplingParams["top_k"].(float64); ok {
topKInt := int(topK)
chatReq.TopK = &topKInt
}
requestCtx := context.Background()
// Use non-streaming completion for /generate endpoint
resp, err := h.service.Client().CreateChatCompletion(requestCtx, chatReq)
if err != nil {
h.logger.Error("Failed to create completion",
zap.Error(err),
)
utils.RespondError(ctx, 500, fmt.Sprintf("Failed to create completion: %v", err), "server_error")
return
}
// Convert to SGLang /generate response format
// meta_info must match SGLang's expected format with completion_tokens at top level
finishReason := resp.Choices[0].FinishReason
if finishReason == "" {
finishReason = "stop"
}
response := map[string]interface{}{
"text": resp.Choices[0].Message.Content,
"meta_info": map[string]interface{}{
"id": resp.ID,
"finish_reason": finishReason,
"prompt_tokens": resp.Usage.PromptTokens,
"completion_tokens": resp.Usage.CompletionTokens,
"cached_tokens": 0, // Not available from chat completion API
"weight_version": "", // Not available from chat completion API
},
}
ctx.SetStatusCode(200)
ctx.SetContentType("application/json")
jsonData, _ := json.Marshal(response)
ctx.Write(jsonData)
}

View File

@@ -0,0 +1,33 @@
package handlers
import (
"encoding/json"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
// HealthHandler handles health check requests
type HealthHandler struct {
logger *zap.Logger
}
// NewHealthHandler creates a new health handler
func NewHealthHandler(logger *zap.Logger) *HealthHandler {
return &HealthHandler{
logger: logger,
}
}
// Check handles GET /health
func (h *HealthHandler) Check(ctx *fasthttp.RequestCtx) {
ctx.SetStatusCode(200)
ctx.SetContentType("application/json")
response := map[string]string{
"status": "ok",
}
jsonData, _ := json.Marshal(response)
ctx.Write(jsonData)
}

View File

@@ -0,0 +1,67 @@
package handlers
import (
"encoding/json"
"github.com/valyala/fasthttp"
"go.uber.org/zap"
)
// ModelsHandler handles model list requests
type ModelsHandler struct {
logger *zap.Logger
tokenizerPath string
}
// NewModelsHandler creates a new models handler
func NewModelsHandler(logger *zap.Logger, tokenizerPath string) *ModelsHandler {
return &ModelsHandler{
logger: logger,
tokenizerPath: tokenizerPath,
}
}
// List handles GET /v1/models
func (h *ModelsHandler) List(ctx *fasthttp.RequestCtx) {
// Return a default model for OpenAI compatibility
ctx.SetStatusCode(200)
ctx.SetContentType("application/json")
response := map[string]interface{}{
"object": "list",
"data": []map[string]interface{}{
{
"id": "default",
"object": "model",
"created": 1677610602,
"owned_by": "sglang",
},
},
}
jsonData, _ := json.Marshal(response)
ctx.Write(jsonData)
}
// GetModelInfo handles GET /get_model_info
// Returns model information compatible with SGLang RuntimeEndpoint
func (h *ModelsHandler) GetModelInfo(ctx *fasthttp.RequestCtx) {
ctx.SetStatusCode(200)
ctx.SetContentType("application/json")
// Return model info compatible with SGLang RuntimeEndpoint expectations
response := map[string]interface{}{
"model_path": h.tokenizerPath, // Use tokenizer path as model path
"tokenizer_path": h.tokenizerPath,
"is_generation": true,
"preferred_sampling_params": "",
"weight_version": "",
"has_image_understanding": false,
"has_audio_understanding": false,
"model_type": "",
"architectures": nil,
}
jsonData, _ := json.Marshal(response)
ctx.Write(jsonData)
}

View File

@@ -0,0 +1,67 @@
package logger
import (
"os"
"path/filepath"
"time"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"gopkg.in/natefinch/lumberjack.v2"
)
// Init initializes the logger with file and console output
func Init(logDir, logLevel string) (*zap.Logger, error) {
// Ensure log directory exists
if err := os.MkdirAll(logDir, 0755); err != nil {
return nil, err
}
// Parse log level
var level zapcore.Level
if err := level.UnmarshalText([]byte(logLevel)); err != nil {
level = zapcore.InfoLevel
}
// Create log file path with date
logFile := filepath.Join(logDir, "oai_server-"+time.Now().Format("2006-01-02")+".log")
// File writer with rotation
fileWriter := zapcore.AddSync(&lumberjack.Logger{
Filename: logFile,
MaxSize: 100, // megabytes
MaxBackups: 10,
MaxAge: 30, // days
Compress: true,
})
// Console writer
consoleWriter := zapcore.AddSync(os.Stdout)
// Encoder config
encoderConfig := zap.NewProductionEncoderConfig()
encoderConfig.TimeKey = "timestamp"
encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder
// Create cores
fileCore := zapcore.NewCore(
zapcore.NewJSONEncoder(encoderConfig),
fileWriter,
level,
)
consoleCore := zapcore.NewCore(
zapcore.NewConsoleEncoder(encoderConfig),
consoleWriter,
level,
)
// Combine cores
core := zapcore.NewTee(fileCore, consoleCore)
// Create logger
logger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel))
return logger, nil
}

View File

@@ -0,0 +1,116 @@
// OpenAI-compatible chat server using SGLang Go SDK and fasthttp framework
package main
import (
"fmt"
"net/http"
"os"
_ "net/http/pprof" // Enable pprof endpoints
"github.com/valyala/fasthttp"
"go.uber.org/zap"
"oai_server/config"
"oai_server/handlers"
"oai_server/logger"
"oai_server/service"
)
// Version information (set at build time via ldflags)
var (
Version = "dev"
BuildTime = "unknown"
GitCommit = "unknown"
)
func main() {
// Load configuration
cfg := config.Load()
// Initialize logger
appLogger, err := logger.Init(cfg.LogDir, cfg.LogLevel)
if err != nil {
panic(fmt.Sprintf("Failed to initialize logger: %v", err))
}
defer appLogger.Sync()
appLogger.Info("Starting OpenAI-compatible server",
zap.String("endpoint", cfg.Endpoint),
zap.String("tokenizer", cfg.TokenizerPath),
zap.String("port", cfg.Port),
)
// Initialize SGLang service
sglangService, err := service.NewSGLangService(cfg.Endpoint, cfg.TokenizerPath)
if err != nil {
appLogger.Fatal("Failed to create SGLang client", zap.Error(err))
}
defer sglangService.Close()
appLogger.Info("SGLang client created successfully")
// Enable pprof if requested
if os.Getenv("PPROF_ENABLED") == "true" {
pprofPort := os.Getenv("PPROF_PORT")
if pprofPort == "" {
pprofPort = "6060"
}
go func() {
pprofAddr := ":" + pprofPort
appLogger.Info("Starting pprof server", zap.String("address", pprofAddr))
if err := http.ListenAndServe(pprofAddr, nil); err != nil {
appLogger.Error("pprof server failed", zap.Error(err))
}
}()
appLogger.Info("pprof enabled", zap.String("port", pprofPort), zap.String("endpoint", fmt.Sprintf("http://localhost:%s/debug/pprof/", pprofPort)))
}
// Initialize handlers
healthHandler := handlers.NewHealthHandler(appLogger)
modelsHandler := handlers.NewModelsHandler(appLogger, cfg.TokenizerPath)
chatHandler := handlers.NewChatHandler(appLogger, sglangService)
// Setup fasthttp router
router := func(ctx *fasthttp.RequestCtx) {
path := string(ctx.Path())
method := string(ctx.Method())
switch {
case method == "GET" && path == "/health":
healthHandler.Check(ctx)
case method == "GET" && path == "/v1/models":
modelsHandler.List(ctx)
case method == "GET" && path == "/get_model_info":
modelsHandler.GetModelInfo(ctx)
case method == "POST" && path == "/v1/chat/completions":
chatHandler.HandleChatCompletion(ctx)
case (method == "POST" || method == "PUT") && path == "/generate":
chatHandler.HandleGenerate(ctx)
default:
ctx.Error("Not Found", fasthttp.StatusNotFound)
}
}
// Start server
serverAddr := ":" + cfg.Port
baseURL := fmt.Sprintf("http://localhost:%s", cfg.Port)
appLogger.Info("Server starting",
zap.String("address", serverAddr),
zap.String("base_url", baseURL),
)
// Print available HTTP endpoints (similar to FastAPI startup)
appLogger.Info("Available HTTP endpoints:")
appLogger.Info(fmt.Sprintf(" GET %s/health", baseURL))
appLogger.Info(fmt.Sprintf(" GET %s/v1/models", baseURL))
appLogger.Info(fmt.Sprintf(" GET %s/get_model_info", baseURL))
appLogger.Info(fmt.Sprintf(" POST %s/v1/chat/completions", baseURL))
appLogger.Info(fmt.Sprintf(" POST %s/generate", baseURL))
appLogger.Info(fmt.Sprintf("Application startup complete. Listening on %s", baseURL))
if err := fasthttp.ListenAndServe(serverAddr, router); err != nil {
appLogger.Fatal("Server failed", zap.Error(err))
}
}

View File

@@ -0,0 +1,14 @@
package models
// ChatRequest represents an OpenAI-compatible chat completion request
type ChatRequest struct {
Model string `json:"model" binding:"required"`
Messages []map[string]string `json:"messages" binding:"required"`
Stream bool `json:"stream,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
MaxTokens *int `json:"max_tokens,omitempty"` // OpenAI API standard field
MaxCompletionTokens *int `json:"max_completion_tokens,omitempty"` // SGLang-specific field (used by bench_serving.py)
Tools []map[string]interface{} `json:"tools,omitempty"`
ToolChoice interface{} `json:"tool_choice,omitempty"`
}

View File

@@ -0,0 +1,111 @@
#!/bin/bash
# OpenAI-compatible server runner
# Usage: ./run.sh [tokenizer_path] [endpoint] [port] [--profile] [--pprof-port PORT]
#
# Options:
# --profile Enable pprof profiling (default port: 6060)
# --pprof-port PORT Set pprof port (default: 6060, requires --profile)
# Set library path for Rust FFI library
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BINDINGS_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
LIB_DIR="${BINDINGS_DIR}/lib"
if [ ! -d "$LIB_DIR" ]; then
echo "Error: Library directory not found at $LIB_DIR"
echo "Please run 'make lib' first to build and export the library"
exit 1
fi
# Get Python LDFLAGS (needed for Rust FFI that depends on Python)
PYTHON_LDFLAGS=$(python3-config --ldflags --embed 2>/dev/null || python3-config --ldflags 2>/dev/null || echo "")
# Set CGO_LDFLAGS to link with the Rust library
# Note: -lsgl_model_gateway_go and -ldl are already in the #cgo directive in internal/ffi/client.go
# We only need to add the library path (-L) and Python flags
export CGO_LDFLAGS="-L${LIB_DIR} ${PYTHON_LDFLAGS}"
# macOS uses DYLD_LIBRARY_PATH, Linux uses LD_LIBRARY_PATH
if [[ "$OSTYPE" == "darwin"* ]]; then
export DYLD_LIBRARY_PATH="${LIB_DIR}:${DYLD_LIBRARY_PATH}"
else
export LD_LIBRARY_PATH="${LIB_DIR}:${LD_LIBRARY_PATH}"
fi
# Parse arguments
ENABLE_PROFILE=false
PPROF_PORT="6060"
TOKENIZER_PATH=""
ENDPOINT=""
PORT=""
while [[ $# -gt 0 ]]; do
case $1 in
--profile)
ENABLE_PROFILE=true
shift
;;
--pprof-port)
ENABLE_PROFILE=true
PPROF_PORT="$2"
shift 2
;;
*)
if [[ -z "$TOKENIZER_PATH" ]]; then
TOKENIZER_PATH="$1"
elif [[ -z "$ENDPOINT" ]]; then
ENDPOINT="$1"
elif [[ -z "$PORT" ]]; then
PORT="$1"
fi
shift
;;
esac
done
# Default configuration
DEFAULT_TOKENIZER_PATH="${SGL_TOKENIZER_PATH:-../tokenizer}"
DEFAULT_ENDPOINT="${SGL_GRPC_ENDPOINT:-grpc://localhost:20000}"
DEFAULT_PORT="${PORT:-8080}"
TOKENIZER_PATH="${TOKENIZER_PATH:-${DEFAULT_TOKENIZER_PATH}}"
ENDPOINT="${ENDPOINT:-${DEFAULT_ENDPOINT}}"
PORT="${PORT:-${DEFAULT_PORT}}"
echo "Running OpenAI-compatible server..."
echo "Library path: ${LIB_DIR}"
echo "Tokenizer: $TOKENIZER_PATH"
echo "Endpoint: $ENDPOINT"
echo "Port: $PORT"
echo "Client Mode: gRPC (default)"
echo "FFI Postprocessing: ENABLED (normal mode)"
echo "FFI Preprocessing: ENABLED (normal mode)"
if [[ "$ENABLE_PROFILE" == "true" ]]; then
echo "Profiling: enabled (port: $PPROF_PORT)"
echo " pprof endpoint: http://localhost:$PPROF_PORT/debug/pprof/"
export PPROF_ENABLED=true
export PPROF_PORT="$PPROF_PORT"
else
echo "Profiling: disabled"
fi
echo ""
# Change to script directory
cd "$(dirname "${BASH_SOURCE[0]}")"
# Ensure Go module is properly initialized
if [ ! -f "go.mod" ]; then
echo "Error: go.mod not found in $(pwd)"
exit 1
fi
# Ensure Go modules are enabled
export GO111MODULE=on
# Sync Go module dependencies
echo "Syncing Go module dependencies..."
go mod tidy
# Run the server (use ./main.go to ensure module context is correct)
SGL_TOKENIZER_PATH="$TOKENIZER_PATH" SGL_GRPC_ENDPOINT="$ENDPOINT" PORT="$PORT" go run ./main.go

View File

@@ -0,0 +1,554 @@
#!/bin/bash
# TPOT performance bottleneck analysis script
# Specifically designed to analyze why Go Router is twice as slow as Rust Router
#
# Usage:
# ./scripts/analyze_tpot.sh [options]
#
# Options:
# --duration SECONDS CPU profile duration (default: 60)
# --requests NUM Number of requests (default: 100)
# --concurrency NUM Concurrency level (default: 20)
# --pprof-port PORT pprof port (default: 6060)
# --server-url URL Server URL (default: http://localhost:8080)
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
PROFILE_DIR="${PROJECT_ROOT}/profiles"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
OUTPUT_DIR="${PROFILE_DIR}/tpot_analysis_${TIMESTAMP}"
# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m'
# Default values
DURATION=${DURATION:-60}
NUM_REQUESTS=${NUM_REQUESTS:-100}
CONCURRENCY=${CONCURRENCY:-20}
PPROF_PORT=${PPROF_PORT:-6060}
SERVER_URL=${SERVER_URL:-http://localhost:8080}
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
--duration)
DURATION="$2"
shift 2
;;
--requests)
NUM_REQUESTS="$2"
shift 2
;;
--concurrency)
CONCURRENCY="$2"
shift 2
;;
--pprof-port)
PPROF_PORT="$2"
shift 2
;;
--server-url)
SERVER_URL="$2"
shift 2
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
mkdir -p "$OUTPUT_DIR"
# Check for graphviz (optional, needed for some pprof visualizations)
HAS_GRAPHVIZ=false
if command -v dot >/dev/null 2>&1; then
HAS_GRAPHVIZ=true
fi
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}TPOT Performance Bottleneck Analysis${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo "Configuration:"
echo " Duration: ${DURATION}s"
echo " Requests: $NUM_REQUESTS"
echo " Concurrency: $CONCURRENCY"
echo " Server URL: $SERVER_URL"
echo " pprof Port: $PPROF_PORT"
echo " Output Dir: $OUTPUT_DIR"
if [ "$HAS_GRAPHVIZ" = "false" ]; then
echo ""
echo -e "${YELLOW}Note: graphviz not found. Some pprof visualizations may not work.${NC}"
echo -e "${YELLOW}To install graphviz:${NC}"
echo -e "${YELLOW} macOS: brew install graphviz${NC}"
echo -e "${YELLOW} Ubuntu: sudo apt-get install graphviz${NC}"
echo -e "${YELLOW} CentOS: sudo yum install graphviz${NC}"
echo -e "${YELLOW}Text reports will still be generated without graphviz.${NC}"
fi
echo ""
# Check if server is running
echo -e "${YELLOW}[Check] Verifying server is running...${NC}"
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
echo -e "${RED}Error: Server not responding at ${SERVER_URL}${NC}"
echo ""
echo "Please start the server first with profiling enabled:"
echo " ./run.sh --profile --pprof-port $PPROF_PORT"
echo " or"
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT make run"
exit 1
fi
echo -e "${GREEN}✓ Server is running${NC}"
echo ""
# Check if pprof is enabled
echo -e "${YELLOW}[Check] Verifying pprof is enabled...${NC}"
if ! curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
echo -e "${RED}Error: pprof not accessible at http://localhost:${PPROF_PORT}/debug/pprof/${NC}"
echo ""
echo "Please start the server with profiling enabled:"
echo " ./run.sh --profile --pprof-port $PPROF_PORT"
exit 1
fi
echo -e "${GREEN}✓ pprof is enabled${NC}"
echo ""
# ============================================
# Step 1: Collect baseline profiles
# ============================================
echo -e "${GREEN}[Step 1/8] Collecting baseline profiles...${NC}"
# Baseline memory
go tool pprof -proto -output="${OUTPUT_DIR}/heap_before.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
# Baseline goroutine
go tool pprof -proto -output="${OUTPUT_DIR}/goroutine_before.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" > /dev/null 2>&1 || true
echo -e "${GREEN}✓ Baseline profiles collected${NC}"
echo ""
# ============================================
# Step 2: Start CPU profile collection
# ============================================
echo -e "${GREEN}[Step 2/8] Starting CPU profile collection (${DURATION}s)...${NC}"
go tool pprof -proto -output="${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}" &
CPU_PID=$!
sleep 2
echo -e "${GREEN}✓ CPU profile collection started${NC}"
echo ""
# ============================================
# Step 3: Run load test with streaming requests
# ============================================
echo -e "${GREEN}[Step 3/8] Running load test ($NUM_REQUESTS streaming requests, concurrency=$CONCURRENCY)...${NC}"
# Function to run a single streaming request
run_streaming_request() {
local request_id=$1
local start_time=$(date +%s)
local start_nanos=$(date +%N 2>/dev/null || echo "000000000")
curl -N -s -X POST "${SERVER_URL}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"default\",
\"messages\": [{\"role\": \"user\", \"content\": \"Write a 500-word story with character dialogue and scene descriptions\"}],
\"stream\": true,
\"max_tokens\": 300,
\"temperature\": 0.7
}" > /dev/null
local end_time=$(date +%s)
local end_nanos=$(date +%N 2>/dev/null || echo "000000000")
local duration=$((end_time - start_time))
echo "$duration" >> "${OUTPUT_DIR}/request_times.txt"
}
# Run requests with controlled concurrency
# Use a temporary file to track job PIDs to avoid conflicts with CPU_PID
JOB_PIDS_FILE="${OUTPUT_DIR}/.job_pids_$$"
> "$JOB_PIDS_FILE"
for i in $(seq 1 $NUM_REQUESTS); do
# Wait if we've reached concurrency limit
while [ $(wc -l < "$JOB_PIDS_FILE" 2>/dev/null || echo 0) -ge $CONCURRENCY ]; do
# Check and remove completed jobs
while IFS= read -r pid; do
if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then
# Process completed, remove from file
grep -v "^${pid}$" "$JOB_PIDS_FILE" > "${JOB_PIDS_FILE}.tmp" && \
mv "${JOB_PIDS_FILE}.tmp" "$JOB_PIDS_FILE" || true
fi
done < "$JOB_PIDS_FILE"
sleep 0.1
done
# Start new request
run_streaming_request $i &
echo $! >> "$JOB_PIDS_FILE"
# Progress indicator
if [ $((i % 10)) -eq 0 ]; then
echo " Progress: $i/$NUM_REQUESTS requests sent..."
fi
done
# Wait for all remaining jobs (excluding CPU_PID)
while IFS= read -r pid; do
if [ -n "$pid" ] && [ "$pid" != "$CPU_PID" ]; then
wait "$pid" 2>/dev/null || true
fi
done < "$JOB_PIDS_FILE"
# Clean up
rm -f "$JOB_PIDS_FILE" "${JOB_PIDS_FILE}.tmp" 2>/dev/null || true
echo -e "${GREEN}✓ Load test completed${NC}"
echo ""
# ============================================
# Step 4: Wait for CPU profile to complete
# ============================================
echo -e "${GREEN}[Step 4/8] Waiting for CPU profile to complete...${NC}"
# Wait for the process, but handle the case where it might have already completed
if kill -0 $CPU_PID 2>/dev/null; then
wait $CPU_PID 2>/dev/null || true
else
# Process already completed, just wait a bit to ensure file is written
sleep 1
fi
echo -e "${GREEN}✓ CPU profile collection completed${NC}"
echo ""
# ============================================
# Step 5: Collect final profiles
# ============================================
echo -e "${GREEN}[Step 5/8] Collecting final profiles...${NC}"
# Final memory
go tool pprof -proto -output="${OUTPUT_DIR}/heap_after.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
# Final goroutine
go tool pprof -proto -output="${OUTPUT_DIR}/goroutine_after.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" > /dev/null 2>&1 || true
# Mutex profile
go tool pprof -proto -output="${OUTPUT_DIR}/mutex.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/mutex" > /dev/null 2>&1 || true
# Block profile
go tool pprof -proto -output="${OUTPUT_DIR}/block.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/block" > /dev/null 2>&1 || true
echo -e "${GREEN}✓ Final profiles collected${NC}"
echo ""
# ============================================
# Step 6: Generate analysis reports
# ============================================
echo -e "${GREEN}[Step 6/8] Generating analysis reports...${NC}"
# CPU analysis
echo " Generating CPU reports..."
go tool pprof -top -cum "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/01_cpu_top_cum.txt" 2>&1 || true
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/02_cpu_top_flat.txt" 2>&1 || true
# Memory analysis
echo " Generating memory reports..."
if [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
go tool pprof -top -alloc_space "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/03_memory_alloc_space.txt" 2>&1 || true
go tool pprof -top -alloc_objects "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/04_memory_alloc_objects.txt" 2>&1 || true
go tool pprof -top -inuse_space "${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/05_memory_inuse_space.txt" 2>&1 || true
fi
# Memory growth
if [ -f "${OUTPUT_DIR}/heap_before.pb.gz" ] && [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
go tool pprof -top -base="${OUTPUT_DIR}/heap_before.pb.gz" \
"${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/06_memory_growth.txt" 2>&1 || true
fi
# FFI/CGO analysis
echo " Analyzing FFI/CGO calls..."
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
grep -iE "(block_on|CGO|FFI|ffi|runtime\.cgo|_Cfunc)" > "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt" || \
echo "No FFI/CGO related functions found" > "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
# JSON serialization analysis
echo " Analyzing JSON serialization..."
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
grep -iE "(json|Marshal|Unmarshal|Encode|Decode|sonic|jsoniter)" > "${OUTPUT_DIR}/08_json_analysis.txt" || \
echo "No JSON related functions found" > "${OUTPUT_DIR}/08_json_analysis.txt"
# Goroutine analysis
if [ -f "${OUTPUT_DIR}/goroutine_after.pb.gz" ]; then
echo " Analyzing goroutines..."
go tool pprof -top "${OUTPUT_DIR}/goroutine_after.pb.gz" > "${OUTPUT_DIR}/09_goroutine_analysis.txt" 2>&1 || true
fi
# Mutex analysis
if [ -f "${OUTPUT_DIR}/mutex.pb.gz" ]; then
echo " Analyzing mutex contention..."
go tool pprof -top "${OUTPUT_DIR}/mutex.pb.gz" > "${OUTPUT_DIR}/10_mutex_analysis.txt" 2>&1 || true
fi
# Block analysis
if [ -f "${OUTPUT_DIR}/block.pb.gz" ]; then
echo " Analyzing blocking operations..."
go tool pprof -top "${OUTPUT_DIR}/block.pb.gz" > "${OUTPUT_DIR}/11_block_analysis.txt" 2>&1 || true
fi
# Request timing statistics
if [ -f "${OUTPUT_DIR}/request_times.txt" ] && [ -s "${OUTPUT_DIR}/request_times.txt" ]; then
echo " Calculating request timing statistics..."
{
echo "Request Timing Statistics"
echo "========================"
echo ""
echo "Total requests: $(wc -l < "${OUTPUT_DIR}/request_times.txt" | tr -d ' ')"
echo ""
awk '{
sum+=$1
sumsq+=$1*$1
if(NR==1 || $1<min) min=$1
if(NR==1 || $1>max) max=$1
} END {
if(NR > 0) {
mean=sum/NR
variance=(sumsq/NR - mean*mean)
stddev=sqrt(variance)
print "Min: " min "s"
print "Max: " max "s"
print "Mean: " mean "s"
print "StdDev: " stddev "s"
}
}' "${OUTPUT_DIR}/request_times.txt"
} > "${OUTPUT_DIR}/12_request_timing.txt"
fi
echo -e "${GREEN}✓ Analysis reports generated${NC}"
echo ""
# ============================================
# Step 7: Generate summary report
# ============================================
echo -e "${GREEN}[Step 7/8] Generating summary report...${NC}"
SUMMARY_FILE="${OUTPUT_DIR}/00_SUMMARY.md"
cat > "$SUMMARY_FILE" <<EOF
# TPOT Performance Analysis Summary
**Analysis Date:** $(date)
**Duration:** ${DURATION}s
**Requests:** $NUM_REQUESTS
**Concurrency:** $CONCURRENCY
## Key Findings
### 1. CPU Hotspots (Top 10 Cumulative Time)
\`\`\`
$(head -15 "${OUTPUT_DIR}/01_cpu_top_cum.txt" | tail -10)
\`\`\`
### 2. CPU Hotspots (Top 10 Flat Time)
\`\`\`
$(head -15 "${OUTPUT_DIR}/02_cpu_top_flat.txt" | tail -10)
\`\`\`
### 3. FFI/CGO Overhead
\`\`\`
$(cat "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt")
\`\`\`
### 4. JSON Serialization Overhead
\`\`\`
$(cat "${OUTPUT_DIR}/08_json_analysis.txt")
\`\`\`
### 5. Memory Allocation (Top 10 by Space)
\`\`\`
$(head -15 "${OUTPUT_DIR}/03_memory_alloc_space.txt" | tail -10)
\`\`\`
### 6. Memory Allocation (Top 10 by Objects)
\`\`\`
$(head -15 "${OUTPUT_DIR}/04_memory_alloc_objects.txt" | tail -10)
\`\`\`
### 7. Mutex Contention
\`\`\`
$(head -15 "${OUTPUT_DIR}/10_mutex_analysis.txt" | tail -10 2>/dev/null || echo "No significant mutex contention detected")
\`\`\`
### 8. Blocking Operations
\`\`\`
$(head -15 "${OUTPUT_DIR}/11_block_analysis.txt" | tail -10 2>/dev/null || echo "No significant blocking detected")
\`\`\`
## Performance Bottlenecks Identified
### High Priority Issues
1. **FFI/CGO Overhead**
- Check: \`cat ${OUTPUT_DIR}/07_ffi_cgo_analysis.txt\`
- Impact: FFI calls add overhead compared to native Rust code
- Recommendation: Minimize FFI calls, batch operations
2. **JSON Serialization**
- Check: \`cat ${OUTPUT_DIR}/08_json_analysis.txt\`
- Impact: JSON marshaling/unmarshaling can be expensive
- Recommendation: Use faster JSON library (jsoniter), reduce serialization frequency
3. **Memory Allocations**
- Check: \`cat ${OUTPUT_DIR}/03_memory_alloc_space.txt\`
- Impact: Frequent allocations cause GC pressure
- Recommendation: Use object pools, pre-allocate buffers
### Medium Priority Issues
4. **Goroutine Overhead**
- Check: \`cat ${OUTPUT_DIR}/09_goroutine_analysis.txt\`
- Impact: Too many goroutines can cause scheduling overhead
- Recommendation: Limit goroutine count, use worker pools
5. **Lock Contention**
- Check: \`cat ${OUTPUT_DIR}/10_mutex_analysis.txt\`
- Impact: Lock contention reduces parallelism
- Recommendation: Reduce lock granularity, use lock-free structures
## Comparison with Rust Router
### Expected Differences
1. **FFI Overhead**: Go → Rust FFI calls add ~100-500ns per call
2. **GC Overhead**: Go's GC can cause pauses (usually <1ms)
3. **JSON Library**: Go's standard library is slower than Rust's serde
4. **Memory Layout**: Go's GC affects cache locality
### Optimization Opportunities
1. **Reduce FFI Calls**
- Batch token processing
- Use async FFI (if possible)
- Cache frequently used FFI results
2. **Optimize JSON**
- Use jsoniter (already implemented)
- Pre-allocate JSON buffers
- Reduce serialization frequency
3. **Memory Management**
- Use sync.Pool for frequently allocated objects
- Pre-allocate slices with known capacity
- Avoid unnecessary string copies
4. **Concurrency**
- Use worker pools instead of spawning goroutines per request
- Limit concurrent FFI calls
- Use channels efficiently
## Next Steps
1. Review detailed reports in this directory
2. Use interactive pprof: \`go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz\`
3. Compare with Rust router profiles (if available)
4. Implement optimizations based on findings
5. Re-run analysis to measure improvements
## Files Generated
- \`00_SUMMARY.md\` - This summary
- \`01_cpu_top_cum.txt\` - CPU top functions (cumulative)
- \`02_cpu_top_flat.txt\` - CPU top functions (flat)
- \`03_memory_alloc_space.txt\` - Memory allocation by space
- \`04_memory_alloc_objects.txt\` - Memory allocation by objects
- \`05_memory_inuse_space.txt\` - Memory in use by space
- \`06_memory_growth.txt\` - Memory growth during test
- \`07_ffi_cgo_analysis.txt\` - FFI/CGO overhead analysis
- \`08_json_analysis.txt\` - JSON serialization analysis
- \`09_goroutine_analysis.txt\` - Goroutine analysis
- \`10_mutex_analysis.txt\` - Mutex contention analysis
- \`11_block_analysis.txt\` - Blocking operations analysis
- \`12_request_timing.txt\` - Request timing statistics
- \`*.pb.gz\` - Raw profile files for interactive analysis
EOF
echo -e "${GREEN}✓ Summary report generated${NC}"
echo ""
# ============================================
# Step 8: Display summary
# ============================================
echo -e "${GREEN}[Step 8/8] Analysis Complete!${NC}"
echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Summary${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo -e "${YELLOW}Top CPU Hotspots (Cumulative):${NC}"
head -12 "${OUTPUT_DIR}/01_cpu_top_cum.txt" | tail -10
echo ""
echo -e "${YELLOW}FFI/CGO Overhead:${NC}"
cat "${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
echo ""
echo -e "${YELLOW}JSON Serialization Overhead:${NC}"
cat "${OUTPUT_DIR}/08_json_analysis.txt"
echo ""
echo -e "${YELLOW}Top Memory Allocations:${NC}"
head -12 "${OUTPUT_DIR}/03_memory_alloc_space.txt" | tail -10
echo ""
if [ -f "${OUTPUT_DIR}/12_request_timing.txt" ]; then
echo -e "${YELLOW}Request Timing:${NC}"
cat "${OUTPUT_DIR}/12_request_timing.txt"
echo ""
fi
echo -e "${GREEN}========================================${NC}"
echo ""
echo -e "${BLUE}Detailed Reports:${NC}"
echo " Summary: cat ${OUTPUT_DIR}/00_SUMMARY.md"
echo " CPU (cum): cat ${OUTPUT_DIR}/01_cpu_top_cum.txt"
echo " CPU (flat): cat ${OUTPUT_DIR}/02_cpu_top_flat.txt"
echo " FFI/CGO: cat ${OUTPUT_DIR}/07_ffi_cgo_analysis.txt"
echo " JSON: cat ${OUTPUT_DIR}/08_json_analysis.txt"
echo " Memory: cat ${OUTPUT_DIR}/03_memory_alloc_space.txt"
echo ""
echo -e "${BLUE}Interactive Analysis:${NC}"
echo " Run: go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz"
echo " Then visit:"
echo " - http://localhost:8081/ui/flamegraph (Flame Graph - no graphviz needed)"
echo " - http://localhost:8081/ui/top (Top Functions - no graphviz needed)"
if [ "$HAS_GRAPHVIZ" = "true" ]; then
echo " - http://localhost:8081/ui/graph (Call Graph - requires graphviz)"
else
echo " - http://localhost:8081/ui/graph (Call Graph - requires graphviz, not available)"
fi
echo ""
if [ "$HAS_GRAPHVIZ" = "false" ]; then
echo -e "${YELLOW}Note: Install graphviz to enable call graph visualization:${NC}"
echo -e "${YELLOW} macOS: brew install graphviz${NC}"
echo -e "${YELLOW} Ubuntu: sudo apt-get install graphviz${NC}"
echo -e "${YELLOW} CentOS: sudo yum install graphviz${NC}"
echo ""
fi
echo -e "${GREEN}All files saved to: ${OUTPUT_DIR}${NC}"
echo ""

View File

@@ -0,0 +1,215 @@
#!/bin/bash
# pprof performance analysis script
# Used to analyze performance bottlenecks of Go OpenAI server
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
# Configuration
PPROF_PORT=${PPROF_PORT:-6060}
SERVER_PORT=${SERVER_PORT:-8080}
DURATION=${DURATION:-60} # Performance test duration (seconds)
OUTPUT_DIR="./pprof_results"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
# Create output directory
mkdir -p "$OUTPUT_DIR"
echo "=========================================="
echo "pprof Performance Analysis Tool"
echo "=========================================="
echo "PPROF_PORT: $PPROF_PORT"
echo "SERVER_PORT: $SERVER_PORT"
echo "DURATION: ${DURATION}s"
echo "OUTPUT_DIR: $OUTPUT_DIR"
echo ""
# Check if go tool pprof is available
if ! command -v go &> /dev/null; then
echo "Error: go command not found"
exit 1
fi
# Check if server is running
check_server() {
if curl -s "http://localhost:${SERVER_PORT}/health" > /dev/null 2>&1; then
return 0
else
return 1
fi
}
# Check if pprof is available
check_pprof() {
if curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
return 0
else
return 1
fi
}
# Start server (if not running)
if ! check_server; then
echo "Server not running, please start the server first:"
echo " export PPROF_ENABLED=true"
echo " export PPROF_PORT=$PPROF_PORT"
echo " ./oai_server"
echo ""
echo "Or use the following command to start:"
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT ./oai_server"
echo ""
read -p "Start server now? (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo "Starting server..."
PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT ./oai_server &
SERVER_PID=$!
echo "Server PID: $SERVER_PID"
# Wait for server to start
echo "Waiting for server to start..."
for i in {1..30}; do
if check_server; then
echo "Server started"
break
fi
sleep 1
done
if ! check_server; then
echo "Error: Server failed to start"
kill $SERVER_PID 2>/dev/null || true
exit 1
fi
else
exit 1
fi
fi
# Check if pprof is available
if ! check_pprof; then
echo "Error: pprof not enabled. Please set environment variables:"
echo " export PPROF_ENABLED=true"
echo " export PPROF_PORT=$PPROF_PORT"
exit 1
fi
echo "Starting to collect performance data..."
echo ""
# 1. CPU Profile (30 seconds)
echo "[1/6] Collecting CPU Profile (30 seconds)..."
go tool pprof -proto -output="$OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30" &
CPU_PID=$!
# 2. Collect Heap Profile simultaneously
echo "[2/6] Collecting Heap Profile..."
go tool pprof -proto -output="$OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/heap" &
HEAP_PID=$!
# 3. Collect Goroutine Profile
echo "[3/6] Collecting Goroutine Profile..."
go tool pprof -proto -output="$OUTPUT_DIR/goroutine_${TIMESTAMP}.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/goroutine" &
GOROUTINE_PID=$!
# 4. Collect Mutex Profile
echo "[4/6] Collecting Mutex Profile..."
go tool pprof -proto -output="$OUTPUT_DIR/mutex_${TIMESTAMP}.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/mutex" &
MUTEX_PID=$!
# 5. Collect Block Profile
echo "[5/6] Collecting Block Profile..."
go tool pprof -proto -output="$OUTPUT_DIR/block_${TIMESTAMP}.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/block" &
BLOCK_PID=$!
# 6. Run performance test (during CPU profile collection)
echo "[6/6] Running performance test..."
echo "Tip: Please use your performance testing tool (curl, ab, wrk, etc.) to send requests to the server"
echo " CPU profile will collect 30 seconds of performance data"
echo ""
# Wait for CPU profile to complete
wait $CPU_PID
echo "CPU Profile collection completed"
# Wait for other profiles
wait $HEAP_PID
wait $GOROUTINE_PID
wait $MUTEX_PID
wait $BLOCK_PID
echo ""
echo "=========================================="
echo "Performance data collection completed!"
echo "=========================================="
echo ""
echo "Generated analysis files:"
ls -lh "$OUTPUT_DIR"/*_${TIMESTAMP}.* 2>/dev/null || true
echo ""
# Generate analysis report
echo "Generating analysis report..."
echo ""
# CPU Top 20
echo "=== CPU Top 20 (sorted by flat time) ===" > "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
go tool pprof -top -cum "$OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
# Heap Top 20
echo "=== Heap Top 20 (sorted by allocation size) ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
go tool pprof -top "$OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
# Goroutine statistics
echo "=== Goroutine Statistics ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
go tool pprof -top "$OUTPUT_DIR/goroutine_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
# Mutex statistics
echo "=== Mutex Wait Time ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
go tool pprof -top "$OUTPUT_DIR/mutex_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
echo "" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
# Block statistics
echo "=== Block Wait Time ===" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
go tool pprof -top "$OUTPUT_DIR/block_${TIMESTAMP}.pb.gz" >> "$OUTPUT_DIR/analysis_${TIMESTAMP}.txt" 2>&1 || true
echo "Analysis report saved to: $OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
echo ""
# Display key information
echo "=========================================="
echo "Key Performance Metrics Summary"
echo "=========================================="
echo ""
echo "View detailed report:"
echo " cat $OUTPUT_DIR/analysis_${TIMESTAMP}.txt"
echo ""
echo "Interactive CPU Profile view:"
echo " go tool pprof $OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz"
echo ""
echo "Interactive Heap Profile view:"
echo " go tool pprof $OUTPUT_DIR/heap_${TIMESTAMP}.pb.gz"
echo ""
echo "Generate flame graph (requires go-torch or pprof):"
echo " go tool pprof -http=:8080 $OUTPUT_DIR/cpu_${TIMESTAMP}.pb.gz"
echo ""
# If server was started, ask if it should be closed
if [ -n "$SERVER_PID" ]; then
read -p "Close server? (y/n) " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
kill $SERVER_PID 2>/dev/null || true
echo "Server closed"
fi
fi

View File

@@ -0,0 +1,52 @@
#!/bin/bash
# Quick pprof analysis script
# Collects 30-second CPU profile and immediately displays top results
set -e
PPROF_PORT=${PPROF_PORT:-6060}
DURATION=${DURATION:-30}
echo "=========================================="
echo "Quick pprof Analysis"
echo "=========================================="
echo "PPROF_PORT: $PPROF_PORT"
echo "DURATION: ${DURATION}s"
echo ""
echo "Tip: During data collection, please send requests to the server"
echo " You can use: ./pprof_test.sh"
echo ""
# Check if pprof is available
if ! curl -s "http://localhost:${PPROF_PORT}/debug/pprof/" > /dev/null 2>&1; then
echo "Error: pprof not enabled. Please set environment variables:"
echo " export PPROF_ENABLED=true"
echo " export PPROF_PORT=$PPROF_PORT"
exit 1
fi
echo "Starting to collect CPU Profile (${DURATION} seconds)..."
echo ""
# Collect CPU profile and directly display top results
go tool pprof -top -cum "http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}"
echo ""
echo "=========================================="
echo "Analysis Complete"
echo "=========================================="
echo ""
echo "More analysis options:"
echo " # Interactive view"
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30"
echo ""
echo " # View heap memory"
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/heap"
echo ""
echo " # View goroutines"
echo " go tool pprof http://localhost:${PPROF_PORT}/debug/pprof/goroutine"
echo ""
echo " # Generate Web UI"
echo " go tool pprof -http=:8080 http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=30"
echo ""

View File

@@ -0,0 +1,87 @@
#!/bin/bash
# Simple performance test script for sending requests while collecting pprof data
set -e
SERVER_URL=${SERVER_URL:-"http://localhost:8080"}
DURATION=${DURATION:-30} # Test duration (seconds)
CONCURRENT=${CONCURRENT:-1} # Number of concurrent requests
echo "=========================================="
echo "Performance Test Script"
echo "=========================================="
echo "SERVER_URL: $SERVER_URL"
echo "DURATION: ${DURATION}s"
echo "CONCURRENT: $CONCURRENT"
echo ""
# Test request JSON
TEST_REQUEST='{
"model": "default",
"messages": [
{"role": "user", "content": "Hello, how are you?"}
],
"stream": true,
"max_tokens": 100
}'
# Check if server is available
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
echo "Error: Server not available (${SERVER_URL}/health)"
exit 1
fi
echo "Starting to send test requests..."
echo ""
# Function to send streaming request
send_stream_request() {
local request_num=$1
local start_time=$(date +%s.%N)
curl -s -N -X POST "${SERVER_URL}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "$TEST_REQUEST" \
> /dev/null 2>&1
local end_time=$(date +%s.%N)
local duration=$(echo "$end_time - $start_time" | bc)
echo "Request $request_num completed, duration: ${duration}s"
}
# Send requests concurrently
if [ "$CONCURRENT" -eq 1 ]; then
# Single-threaded mode: continuously send requests
end_time=$(($(date +%s) + DURATION))
request_count=0
while [ $(date +%s) -lt $end_time ]; do
request_count=$((request_count + 1))
send_stream_request $request_count
done
echo ""
echo "Test completed, sent $request_count requests"
else
# Multi-threaded mode: send requests concurrently
end_time=$(($(date +%s) + DURATION))
request_count=0
while [ $(date +%s) -lt $end_time ]; do
# Start concurrent requests
for i in $(seq 1 $CONCURRENT); do
request_count=$((request_count + 1))
send_stream_request $request_count &
done
# Wait for all requests to complete
wait
# Brief rest to avoid overload
sleep 0.1
done
echo ""
echo "Test completed, sent $request_count requests"
fi

View File

@@ -0,0 +1,140 @@
#!/bin/bash
# TPOT performance analysis script
# Quickly collect and analyze TPOT-related performance data
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
PROFILE_DIR="${PROJECT_ROOT}/profiles"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
OUTPUT_DIR="${PROFILE_DIR}/${TIMESTAMP}"
# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
# Default values
PPROF_PORT=${PPROF_PORT:-6060}
SERVER_URL=${SERVER_URL:-http://localhost:8080}
DURATION=${DURATION:-30}
NUM_REQUESTS=${NUM_REQUESTS:-20}
mkdir -p "$OUTPUT_DIR"
echo -e "${GREEN}TPOT Performance Analysis${NC}"
echo "=========================="
echo "Profile directory: $OUTPUT_DIR"
echo "Duration: ${DURATION}s"
echo "Requests: $NUM_REQUESTS"
echo ""
# Check if server is running
if ! curl -s "${SERVER_URL}/health" > /dev/null 2>&1; then
echo -e "${YELLOW}Warning: Server not responding at ${SERVER_URL}${NC}"
echo "Please start the server first with profiling enabled:"
echo " PPROF_ENABLED=true PPROF_PORT=$PPROF_PORT make run"
exit 1
fi
# Collect baseline memory
echo -e "${GREEN}[1/5] Collecting baseline memory profile...${NC}"
go tool pprof -proto -output="${OUTPUT_DIR}/heap_before.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
# Start CPU profile collection in background
echo -e "${GREEN}[2/5] Starting CPU profile collection (${DURATION}s)...${NC}"
go tool pprof -proto -output="${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/profile?seconds=${DURATION}" &
CPU_PID=$!
# Wait a bit for profile to start
sleep 2
# Run load test
echo -e "${GREEN}[3/5] Running load test ($NUM_REQUESTS requests)...${NC}"
for i in $(seq 1 $NUM_REQUESTS); do
curl -N -s -X POST "${SERVER_URL}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"default\",
\"messages\": [{\"role\": \"user\", \"content\": \"Write a story\"}],
\"stream\": true,
\"max_tokens\": 200
}" > /dev/null &
# Limit concurrency
if [ $((i % 5)) -eq 0 ]; then
wait
fi
done
wait
# Wait for CPU profile to complete
echo -e "${GREEN}[4/5] Waiting for CPU profile to complete...${NC}"
# Wait for the CPU profile process, but handle the case where it's not a child process
if kill -0 $CPU_PID 2>/dev/null; then
# Process is still running, wait for it
while kill -0 $CPU_PID 2>/dev/null; do
sleep 1
done
else
# Process already completed or not found, just wait a bit
sleep 2
fi
# Collect final memory
echo -e "${GREEN}[5/5] Collecting final memory profile...${NC}"
go tool pprof -proto -output="${OUTPUT_DIR}/heap_after.pb.gz" \
"http://localhost:${PPROF_PORT}/debug/pprof/heap" > /dev/null 2>&1 || true
# Generate reports
echo ""
echo -e "${GREEN}Generating reports...${NC}"
# CPU top (cumulative)
go tool pprof -top -cum "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/cpu_top_cum.txt" 2>&1 || true
# CPU top (flat)
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" > "${OUTPUT_DIR}/cpu_top_flat.txt" 2>&1 || true
# Memory growth
if [ -f "${OUTPUT_DIR}/heap_before.pb.gz" ] && [ -f "${OUTPUT_DIR}/heap_after.pb.gz" ]; then
go tool pprof -top -base="${OUTPUT_DIR}/heap_before.pb.gz" \
"${OUTPUT_DIR}/heap_after.pb.gz" > "${OUTPUT_DIR}/heap_growth.txt" 2>&1 || true
fi
# FFI/CGO related
go tool pprof -top "${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz" 2>&1 | \
grep -E "(block_on|CGO|FFI|json|Marshal|Unmarshal)" > "${OUTPUT_DIR}/ffi_related.txt" || \
echo "No FFI/CGO related functions found" > "${OUTPUT_DIR}/ffi_related.txt"
# Summary
echo ""
echo -e "${GREEN}=== Analysis Summary ===${NC}"
echo ""
echo -e "${YELLOW}CPU Top (Cumulative) - Top 10:${NC}"
head -12 "${OUTPUT_DIR}/cpu_top_cum.txt" | tail -10 || true
echo ""
echo -e "${YELLOW}CPU Top (Flat) - Top 10:${NC}"
head -12 "${OUTPUT_DIR}/cpu_top_flat.txt" | tail -10 || true
echo ""
echo -e "${YELLOW}FFI/CGO Related Functions:${NC}"
cat "${OUTPUT_DIR}/ffi_related.txt" || true
echo ""
echo -e "${GREEN}=== Detailed Reports ===${NC}"
echo "CPU (cumulative): cat ${OUTPUT_DIR}/cpu_top_cum.txt"
echo "CPU (flat): cat ${OUTPUT_DIR}/cpu_top_flat.txt"
echo "Memory growth: cat ${OUTPUT_DIR}/heap_growth.txt"
echo "FFI related: cat ${OUTPUT_DIR}/ffi_related.txt"
echo ""
echo -e "${GREEN}=== Interactive Analysis ===${NC}"
echo "Run: go tool pprof -http=:8081 ${OUTPUT_DIR}/cpu_${DURATION}s.pb.gz"
echo "Then visit: http://localhost:8081/ui/flamegraph"
echo ""
echo "Profile files saved to: ${OUTPUT_DIR}"

View File

@@ -0,0 +1,37 @@
package service
import (
sglang "github.com/sglang/sglang-go-grpc-sdk"
)
// SGLangService wraps SGLang client
type SGLangService struct {
client *sglang.Client
}
func NewSGLangService(endpoint, tokenizerPath string) (*SGLangService, error) {
client, err := sglang.NewClient(sglang.ClientConfig{
Endpoint: endpoint,
TokenizerPath: tokenizerPath,
})
if err != nil {
return nil, err
}
return &SGLangService{
client: client,
}, nil
}
// Client returns the underlying SGLang client
func (s *SGLangService) Client() *sglang.Client {
return s.client
}
// Close closes the SGLang client
func (s *SGLangService) Close() error {
if s.client != nil {
return s.client.Close()
}
return nil
}

View File

@@ -0,0 +1,34 @@
package utils
import (
"encoding/json"
"github.com/valyala/fasthttp"
)
// RespondError sends an error response in OpenAI format
func RespondError(ctx *fasthttp.RequestCtx, statusCode int, message, errorType string) {
ctx.SetStatusCode(statusCode)
ctx.SetContentType("application/json")
response := map[string]interface{}{
"error": map[string]interface{}{
"message": message,
"type": errorType,
"code": statusCode,
},
}
jsonData, _ := json.Marshal(response)
ctx.Write(jsonData)
}
// BuildResponseBase builds the base response structure for OpenAI-compatible responses
func BuildResponseBase(id string, created int64, model string) map[string]interface{} {
return map[string]interface{}{
"id": id,
"object": "chat.completion",
"created": created,
"model": model,
}
}